perf(cache): 修复缓存失效 Bug + 集成语义缓存 + 删除 AgenticRAG 死代码

- fix: Query Cache GET/SET Key 不匹配导致命中率始终为 0%
- fix: CACHE_MIN_SCORE 阈值 0.3 对 ChromaDB cosine distance 过于严格
- feat: 在 /rag 端点集成语义缓存(命中时跳过检索+生成,92x 加速)
- refactor: 删除 AgenticRAG 死代码路径(10 个 agentic_*.py,约 1950 行)
- cleanup: 移除 engine.py 死方法、路由死函数、初始化死代码
This commit is contained in:
lacerate551
2026-06-05 21:20:53 +08:00
parent 6deba8fae2
commit 505f79860e
16 changed files with 222 additions and 2052 deletions

View File

@@ -625,7 +625,7 @@ class RAGEngine:
elif len(conditions) > 1:
where_filter = {"$and": conditions}
query_vector = self.embedding_model.encode(query).tolist()
query_vector = self._encode_cached(query).tolist()
recall_k = RERANK_CANDIDATES if (USE_RERANK or USE_HYBRID_SEARCH) else top_k
recall_k = max(recall_k, top_k * RECALL_MULTIPLIER)
@@ -811,6 +811,76 @@ class RAGEngine:
logger.warning(f"FAQ 集合查询失败: {e}")
return get_empty_result()
def _encode_cached(self, text):
"""
带缓存的 embedding 编码
优先从 Embedding CacheLRU读取未命中再调用模型编码并写入缓存。
支持单文本和批量文本输入。
Args:
text: 单个文本字符串 或 文本列表
Returns:
numpy 数组(单文本为一维,批量为二维)
"""
import numpy as _np
# 检查 embedding 缓存是否启用(缓存配置查询结果,避免每次重复导入)
if not hasattr(self, '_emb_cache_enabled'):
self._emb_cache_enabled = True # 默认启用
if CACHE_AVAILABLE:
try:
from config import EMBEDDING_CACHE_ENABLED
self._emb_cache_enabled = EMBEDDING_CACHE_ENABLED
except ImportError:
pass
if not self._emb_cache_enabled:
return self.embedding_model.encode(text)
try:
_cache = get_cache_manager()
except Exception:
return self.embedding_model.encode(text)
# 批量输入
if isinstance(text, list):
try:
cached_embs, missed_indices = _cache.get_embeddings_batch(text)
if missed_indices:
missed_texts = [text[i] for i in missed_indices]
# encode(list) 始终返回 2D ndarray直接按行索引即可
new_embs = self.embedding_model.encode(missed_texts)
if len(missed_indices) == 1:
# 单条时 encode 可能返回 1D需统一处理
if new_embs.ndim == 1:
new_embs = new_embs.reshape(1, -1)
for idx, mi in enumerate(missed_indices):
emb_list = new_embs[idx].tolist()
cached_embs[mi] = emb_list
try:
_cache.set_embedding(text[mi], emb_list)
except Exception:
pass
return _np.array(cached_embs)
except Exception:
# 缓存故障时优雅降级为直接编码
return self.embedding_model.encode(text)
# 单文本输入
cached = _cache.get_embedding(text)
if cached is not None:
return _np.array(cached)
embedding = self.embedding_model.encode(text)
try:
emb_list = embedding.tolist() if hasattr(embedding, 'tolist') else list(embedding)
_cache.set_embedding(text, emb_list)
except Exception:
pass
return embedding
def _search_image_chunks(self, query_vector: list, top_k: int = 5, where_filter: dict = None) -> dict:
"""
独立检索图片切片P0图片独立召回通道
@@ -1387,7 +1457,7 @@ class RAGEngine:
if not target_collections:
return get_empty_result()
query_vector = self.embedding_model.encode(query).tolist()
query_vector = self._encode_cached(query).tolist()
# 扩大召回数量,以便过滤废止切片后仍有足够结果
recall_k = RERANK_CANDIDATES if (USE_RERANK or USE_HYBRID_SEARCH) else top_k
recall_k = max(recall_k, top_k * RECALL_MULTIPLIER)
@@ -1739,12 +1809,12 @@ class RAGEngine:
# === 高精度版:基于语义向量 ===
from core.mmr import mmr_rerank
# 获取查询向量
query_emb = np.array(self.embedding_model.encode(query))
# 获取查询向量(使用 embedding 缓存)
query_emb = np.array(self._encode_cached(query))
# 批量编码所有文档
# 批量编码所有文档(使用 embedding 缓存)
docs_list = results['documents'][0]
all_embeddings = self.embedding_model.encode(docs_list)
all_embeddings = self._encode_cached(docs_list)
# 构建候选列表
candidates = []
@@ -1940,104 +2010,7 @@ class RAGEngine:
reranked[key] = results[key]
return reranked
# ---------------- 安全与工具 ----------------
def check_restricted_documents(self, query, allowed_levels, top_k=3, role=None, department=None):
if not self._initialized:
self.initialize()
if USE_MULTI_KB and self.kb_manager and role and department:
from auth.gateway import get_accessible_collections
all_colls = [c.name for c in self.kb_manager.list_collections()]
accessible = set(get_accessible_collections(role, department, 'read'))
restricted = set(all_colls) - accessible
if not restricted:
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": []}
query_vector = self.embedding_model.encode(query).tolist()
found_sources = set()
top_score = 0.0
for coll_name in restricted:
try:
coll = self.kb_manager.get_collection(coll_name)
if not coll: continue
res = coll.query(query_embeddings=[query_vector], n_results=top_k)
if res['metadatas'] and res['metadatas'][0]:
for meta in res['metadatas'][0]:
found_sources.add(meta.get('source', '未知'))
for dist in (res.get('distances', [[]])[0] or []):
if dist > top_score: top_score = dist
except Exception as e:
logger.debug(f"权限检查遍历失败: {e}")
return {
"has_restricted": len(found_sources) > 0,
"restricted_levels": [c.replace('dept_', '') for c in restricted if True][:3],
"restricted_sources": list(found_sources)[:3],
"top_restricted_score": top_score
}
if not allowed_levels:
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
restricted_levels = {"public", "internal", "confidential", "secret"} - set(allowed_levels)
if not restricted_levels:
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
query_vector = self.embedding_model.encode(query).tolist()
try:
res = self.collection.query(
query_embeddings=[query_vector],
n_results=top_k,
where={"security_level": {"$in": list(restricted_levels)}}
)
docs = res.get('documents', [[]])[0]
if not docs:
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
metas = res.get('metadatas', [[]])[0]
dists = res.get('distances', [[]])[0]
found_levels, found_sources, top_score = set(), set(), 0.0
for meta, dist in zip(metas, dists):
found_levels.add(meta.get('security_level', 'public'))
found_sources.add(meta.get('source', '未知'))
if dist > top_score: top_score = dist
return {
"has_restricted": True,
"restricted_levels": list(found_levels),
"restricted_sources": list(found_sources)[:3],
"top_restricted_score": top_score
}
except Exception as e:
logger.warning(f"受限内容检查失败: {e}")
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
def generate_answer(self, query, context):
"""底层生成答复能力"""
prompt = f"""你是一个严谨的智能助手,请根据以下参考资料回答用户的问题。
...
参考资料:
{context}
用户问题:{query}
请回答:"""
try:
from core.llm_utils import call_llm
result = call_llm(
self.llm_client,
prompt,
MODEL,
temperature=LLM_TEMPERATURE,
max_tokens=LLM_MAX_TOKENS
)
return result or f"调用大模型失败: 返回结果为空"
except Exception as e:
return f"调用大模型失败: {str(e)}"
# ---------------- 流式生成 ----------------
def generate_answer_stream(self, query, context, history=None):
"""