jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 死代码清理与标注

Browse Source 
- 删除 chat_routes.py 中未被使用的 reciprocal_rank_fusion 函数（engine 有同功能方法平替）
- 删除 search_hybrid 中无效的 candidates 参数（未传递给 engine，实际由 RERANK_CANDIDATES 控制）
- 标注 RAG_SEARCH_CANDIDATES 为死代码
- 标注 VECTOR_WEIGHT/BM25_WEIGHT 在动态 RRF 启用时被覆盖
- 标注 llm_budget.py 模块当前未集成到主流程
- 标注 MAX_LLM_CALLS_PER_QUERY/MAX_QUERY_REWRITES 暂不生效
This commit is contained in:
lacerate551
2026-06-17 20:04:49 +08:00
parent 4afa30a946
commit 4fd53f48f9
3 changed files with 385 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
api/chat_routes.py
View File

@@ -493,6 +493,94 @@ def _section_similarity(section_a: str, section_b: str) -> float:
return overlap / union if union > 0 else 0.0
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
min_score: float) -> List[Dict]:
"""
BM25-CrossEncoder 分歧检测救援：当 BM25 排名靠前（top-3）的切片
被 CrossEncoder rerank 压制（分数低于 min_score 或被截断）时，
将其分数提升至保底值，使其通过后续的 min_score 过滤。
适用场景：
- BM25 top-1/top-2 的切片（精确关键词匹配强信号）被 rerank 评分极低
- 正确切片在 rerank 后被截断，根本不在 contexts 中
Args:
contexts: 全部上下文切片
search_result: engine.search_hybrid() 返回的原始结果（含 _bm25_top3）
min_score: 最低分数阈值
Returns:
修改后的 contexts
"""
if not contexts:
return contexts
from config import (
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
CLUSTER_RESCUE_FLOOR,
)
if not BM25_DIVERGENCE_RESCUE_ENABLED:
return contexts
bm25_top3 = search_result.get('_bm25_top3', [])
if not bm25_top3:
return contexts
# 构建 contexts 中已有切片的 ID 索引，用于快速查找
existing_ids = {}
for i, ctx in enumerate(contexts):
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
if chunk_id:
existing_ids[chunk_id] = i
rescued_count = 0
for bm25_item in bm25_top3:
rank = bm25_item.get('rank', 99)
if rank > BM25_DIVERGENCE_MAX_RANK:
continue
bm25_id = bm25_item.get('id')
bm25_meta = bm25_item.get('meta', {})
bm25_doc = bm25_item.get('doc', '')
# 情况 A：切片在 contexts 中但 score < min_score
if bm25_id and bm25_id in existing_ids:
ctx = contexts[existing_ids[bm25_id]]
if ctx.get('score', 0) < min_score:
ctx['score'] = CLUSTER_RESCUE_FLOOR
rescued_count += 1
logger.debug(
f"BM25 分歧救援 (情况A): rank={rank}, "
f"source={bm25_meta.get('source', '')}, "
f"section={bm25_meta.get('section', '')}, "
f"原score→{CLUSTER_RESCUE_FLOOR}"
)
continue
# 情况 B：切片不在 contexts 中（被 rerank 截断或已被过滤）
# 从 _bm25_top3 备份中注入
if bm25_doc and bm25_meta:
injected_ctx = {
'doc': bm25_doc,
'meta': bm25_meta,
'score': CLUSTER_RESCUE_FLOOR,
}
contexts.append(injected_ctx)
rescued_count += 1
logger.debug(
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
f"source={bm25_meta.get('source', '')}, "
f"section={bm25_meta.get('section', '')}"
)
if rescued_count > 0:
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
return contexts
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
min_score: float) -> List[Dict]:
"""
@@ -1752,54 +1840,7 @@ def chat_with_llm(message: str, history: List[Dict] = None, enable_web_search: b
}
def reciprocal_rank_fusion(results_list, weights=None, k=60):
"""
倒数排名融合算法
Args:
results_list: 多个检索结果列表
weights: 各结果权重
k: RRF 参数
Returns:
融合后的排序结果
"""
if weights is None:
weights = [1.0] * len(results_list)
fused_scores = {}
doc_data = {}
for results, weight in zip(results_list, weights):
if not results or not results.get('ids'):
continue
ids = results['ids'][0]
docs = results['documents'][0] if results.get('documents') else [''] * len(ids)
metas = results['metadatas'][0] if results.get('metadatas') else [{}] * len(ids)
distances = results['distances'][0] if results.get('distances') else [0] * len(ids)
for rank, (doc_id, doc, meta, dist) in enumerate(zip(ids, docs, metas, distances)):
if doc_id not in fused_scores:
fused_scores[doc_id] = 0
doc_data[doc_id] = {'doc': doc, 'meta': meta, 'dist': dist}
# RRF 分数
fused_scores[doc_id] += weight / (rank + k)
# 按分数排序
sorted_ids = sorted(fused_scores.keys(), key=lambda x: fused_scores[x], reverse=True)
return {
'ids': sorted_ids,
'documents': [doc_data[i]['doc'] for i in sorted_ids],
'metadatas': [doc_data[i]['meta'] for i in sorted_ids],
'scores': [fused_scores[i] for i in sorted_ids],
'distances': [doc_data[i]['dist'] for i in sorted_ids]
}
def search_hybrid(query: str, top_k: int = 5, candidates: int = 15,
def search_hybrid(query: str, top_k: int = 5,
allowed_levels: list = None, allowed_collections: list = None,
sub_queries: list = None):
"""
@@ -1808,7 +1849,6 @@ def search_hybrid(query: str, top_k: int = 5, candidates: int = 15,
Args:
query: 查询文本
top_k: 返回数量
candidates: 候选数量（用于 RERANK_CANDIDATES，由 config 控制）
allowed_levels: 允许的安全级别
allowed_collections: 允许的向量库列表
sub_queries: 意图分析器生成的子查询列表（对比类查询用）
@@ -1916,7 +1956,7 @@ def rag():
import re
from config import (
IS_PROD, IS_DEV, ENABLE_SESSION,
RAG_SEARCH_TOP_K, RAG_SEARCH_CANDIDATES,
RAG_SEARCH_TOP_K,
MAX_CONTEXT_CHUNKS, MAX_SOURCES_RETURNED,
LLM_TEMPERATURE, LLM_MAX_TOKENS,
MAX_HISTORY_ROUNDS, IMAGE_CONTEXT_HISTORY,
@@ -2158,7 +2198,6 @@ def rag():
search_result = search_hybrid(
retrieval_query,
top_k=RAG_SEARCH_TOP_K,
candidates=RAG_SEARCH_CANDIDATES,
allowed_collections=collections,
sub_queries=sub_queries
)
@@ -2442,6 +2481,9 @@ def rag():
# 4. 构建 prompt（Phase 6：LLM 图片感知）
# Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争
# BM25 分歧检测救援：BM25 top-3 但 rerank 压制的切片
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
# 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时，
# 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景）
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)