sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)
同步内容: - knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback - services/feedback.py: FAQ max_tokens 200→512 - services/session.py: 消息排序增加 id DESC 次级排序 - knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块 - knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略 - knowledge/document.py: 文档删除时清理孤儿文件 - api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援) - core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑 - cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
@@ -293,6 +293,112 @@ def _process_table_doc(doc: str, meta: Dict) -> str:
|
||||
return doc
|
||||
|
||||
|
||||
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
BM25-CrossEncoder 分歧检测救援:当 BM25 排名靠前(top-3)的切片
|
||||
被 CrossEncoder rerank 压制(分数低于 min_score 或被截断)时,
|
||||
将其分数提升至保底值,使其通过后续的 min_score 过滤。
|
||||
|
||||
适用场景:
|
||||
- BM25 top-1/top-2 的切片(精确关键词匹配强信号)被 rerank 评分极低
|
||||
- 正确切片在 rerank 后被截断,根本不在 contexts 中
|
||||
|
||||
Args:
|
||||
contexts: 全部上下文切片
|
||||
search_result: engine.search_hybrid() 返回的原始结果(含 _bm25_top3)
|
||||
min_score: 最低分数阈值
|
||||
|
||||
Returns:
|
||||
修改后的 contexts
|
||||
"""
|
||||
if not contexts:
|
||||
return contexts
|
||||
|
||||
from config import (
|
||||
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
|
||||
CLUSTER_RESCUE_FLOOR,
|
||||
)
|
||||
|
||||
if not BM25_DIVERGENCE_RESCUE_ENABLED:
|
||||
return contexts
|
||||
|
||||
bm25_top3 = search_result.get('_bm25_top3', [])
|
||||
if not bm25_top3:
|
||||
return contexts
|
||||
|
||||
# 构建 contexts 中已有切片的 ID 索引,用于快速查找
|
||||
existing_ids = {}
|
||||
for i, ctx in enumerate(contexts):
|
||||
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
|
||||
if chunk_id:
|
||||
existing_ids[chunk_id] = i
|
||||
|
||||
# 计算 contexts 中的多数 source(用于情况 B 注入校验)
|
||||
# 防止跨文档注入无关切片(如 q013 场景:2.docx 的吸烟场所切片被注入到 1.docx 的查询中)
|
||||
source_counter: Dict[str, int] = {}
|
||||
for ctx in contexts:
|
||||
src = ctx.get('meta', {}).get('source', '')
|
||||
if src:
|
||||
source_counter[src] = source_counter.get(src, 0) + 1
|
||||
majority_source = max(source_counter, key=source_counter.get) if source_counter else None
|
||||
|
||||
rescued_count = 0
|
||||
|
||||
for bm25_item in bm25_top3:
|
||||
rank = bm25_item.get('rank', 99)
|
||||
if rank > BM25_DIVERGENCE_MAX_RANK:
|
||||
continue
|
||||
|
||||
bm25_id = bm25_item.get('id')
|
||||
bm25_meta = bm25_item.get('meta', {})
|
||||
bm25_doc = bm25_item.get('doc', '')
|
||||
|
||||
# 情况 A:切片在 contexts 中但 score < min_score
|
||||
if bm25_id and bm25_id in existing_ids:
|
||||
ctx = contexts[existing_ids[bm25_id]]
|
||||
if ctx.get('score', 0) < min_score:
|
||||
ctx['score'] = CLUSTER_RESCUE_FLOOR
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况A): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}, "
|
||||
f"原score→{CLUSTER_RESCUE_FLOOR}"
|
||||
)
|
||||
continue
|
||||
|
||||
# 情况 B:切片不在 contexts 中(被 rerank 截断或已被过滤)
|
||||
# 从 _bm25_top3 备份中注入,但需校验 source 一致性
|
||||
if bm25_doc and bm25_meta:
|
||||
bm25_source = bm25_meta.get('source', '')
|
||||
# source 一致性校验:只注入与 contexts 多数 source 一致的切片
|
||||
if majority_source and bm25_source and bm25_source != majority_source:
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况B-跳过): rank={rank}, "
|
||||
f"source={bm25_source} != majority={majority_source}, "
|
||||
f"section={bm25_meta.get('section', '')}"
|
||||
)
|
||||
continue
|
||||
injected_ctx = {
|
||||
'doc': bm25_doc,
|
||||
'meta': bm25_meta,
|
||||
'score': CLUSTER_RESCUE_FLOOR,
|
||||
}
|
||||
contexts.append(injected_ctx)
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}"
|
||||
)
|
||||
|
||||
if rescued_count > 0:
|
||||
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
|
||||
|
||||
return contexts
|
||||
|
||||
|
||||
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
@@ -2518,6 +2624,9 @@ def rag():
|
||||
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
||||
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
||||
|
||||
# BM25 分歧检测救援:BM25 top-3 但 rerank 压制的切片
|
||||
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
# 词法匹配救援:当切片文本精确包含查询关键词但 CrossEncoder 评分低时,
|
||||
# 提升分数使其通过 min_score 过滤(适用于独立切片无法触发聚类救援的场景)
|
||||
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
Reference in New Issue
Block a user