init: RAG 知识库服务初始提交
- 后端 API（Flask + Gunicorn） - RAG 引擎（混合检索 + 云端 Reranker + 引用溯源） - 文档解析（MinerU + 多格式支持） - Docker 生产部署配置 - 排除前端项目、敏感配置、模型文件
This commit is contained in:
lacerate551
553
scripts/evaluate_answer.py
Normal file
553
scripts/evaluate_answer.py
Normal file
@@ -0,0 +1,553 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
RAG 答案层评测脚本
|
||||
|
||||
评测指标：
|
||||
- LLM Score: 使用LLM对答案质量打分 (0-1)
|
||||
- ROUGE-L: 最长公共子序列相似度
|
||||
- Semantic Similarity: 语义相似度（使用embedding）
|
||||
|
||||
用法:
|
||||
python scripts/evaluate_answer.py --eval_dataset data/eval_dataset.json
|
||||
python scripts/evaluate_answer.py --topk 5 --output results/answer_results.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
# 添加项目根目录到路径
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnswerEvaluator:
|
||||
"""答案层评测器"""
|
||||
|
||||
def __init__(self, engine=None):
|
||||
"""
|
||||
初始化评测器
|
||||
|
||||
Args:
|
||||
engine: RAG引擎实例，如果为None则自动创建
|
||||
"""
|
||||
self.engine = engine
|
||||
self._llm_client = None
|
||||
self._embedding_model = None
|
||||
|
||||
def _get_engine(self):
|
||||
"""延迟加载引擎"""
|
||||
if self.engine is None:
|
||||
from core.engine import get_engine
|
||||
self.engine = get_engine()
|
||||
return self.engine
|
||||
|
||||
def _get_llm_client(self):
|
||||
"""延迟加载LLM客户端"""
|
||||
if self._llm_client is None:
|
||||
try:
|
||||
from openai import OpenAI
|
||||
from config import API_KEY, BASE_URL, MODEL
|
||||
self._llm_client = OpenAI(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL
|
||||
)
|
||||
self._llm_model = MODEL
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM客户端初始化失败: {e}")
|
||||
self._llm_client = None
|
||||
return self._llm_client
|
||||
|
||||
def _get_embedding_model(self):
|
||||
"""延迟加载embedding模型"""
|
||||
if self._embedding_model is None:
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from config import EMBEDDING_MODEL_PATH
|
||||
self._embedding_model = SentenceTransformer(EMBEDDING_MODEL_PATH)
|
||||
except Exception as e:
|
||||
logger.warning(f"Embedding模型初始化失败: {e}")
|
||||
self._embedding_model = None
|
||||
return self._embedding_model
|
||||
|
||||
def rouge_l(self, generated: str, reference: str) -> float:
|
||||
"""
|
||||
计算 ROUGE-L 分数
|
||||
|
||||
Args:
|
||||
generated: 生成的答案
|
||||
reference: 参考答案
|
||||
|
||||
Returns:
|
||||
ROUGE-L F1 分数 (0-1)
|
||||
"""
|
||||
if not generated or not reference:
|
||||
return 0.0
|
||||
|
||||
# 简单分词（按字符）
|
||||
gen_tokens = list(generated)
|
||||
ref_tokens = list(reference)
|
||||
|
||||
# 计算最长公共子序列长度
|
||||
m, n = len(gen_tokens), len(ref_tokens)
|
||||
if m == 0 or n == 0:
|
||||
return 0.0
|
||||
|
||||
# DP计算LCS
|
||||
dp = [[0] * (n + 1) for _ in range(m + 1)]
|
||||
for i in range(1, m + 1):
|
||||
for j in range(1, n + 1):
|
||||
if gen_tokens[i - 1] == ref_tokens[j - 1]:
|
||||
dp[i][j] = dp[i - 1][j - 1] + 1
|
||||
else:
|
||||
dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
|
||||
|
||||
lcs_len = dp[m][n]
|
||||
|
||||
# 计算Precision和Recall
|
||||
precision = lcs_len / m if m > 0 else 0.0
|
||||
recall = lcs_len / n if n > 0 else 0.0
|
||||
|
||||
# F1分数
|
||||
if precision + recall == 0:
|
||||
return 0.0
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
|
||||
return f1
|
||||
|
||||
def semantic_similarity(self, generated: str, reference: str) -> float:
|
||||
"""
|
||||
计算语义相似度
|
||||
|
||||
Args:
|
||||
generated: 生成的答案
|
||||
reference: 参考答案
|
||||
|
||||
Returns:
|
||||
语义相似度 (0-1)
|
||||
"""
|
||||
model = self._get_embedding_model()
|
||||
if model is None:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
embeddings = model.encode([generated, reference])
|
||||
# 余弦相似度
|
||||
similarity = np.dot(embeddings[0], embeddings[1]) / (
|
||||
np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])
|
||||
)
|
||||
return float(max(0, min(1, similarity))) # 确保在0-1范围内
|
||||
except Exception as e:
|
||||
logger.warning(f"计算语义相似度失败: {e}")
|
||||
return 0.0
|
||||
|
||||
def llm_score(self, query: str, generated: str, reference: str) -> float:
|
||||
"""
|
||||
使用LLM对答案质量打分
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
generated: 生成的答案
|
||||
reference: 参考答案
|
||||
|
||||
Returns:
|
||||
LLM评分 (0-1)
|
||||
"""
|
||||
client = self._get_llm_client()
|
||||
if client is None:
|
||||
logger.warning("LLM客户端不可用，跳过LLM评分")
|
||||
return 0.0
|
||||
|
||||
prompt = f"""请作为专业评测员，对RAG系统的回答质量进行评分。
|
||||
|
||||
【用户问题】
|
||||
{query}
|
||||
|
||||
【参考答案】
|
||||
{reference}
|
||||
|
||||
【系统回答】
|
||||
{generated}
|
||||
|
||||
【评分标准】
|
||||
请从以下维度评分（每项0-10分）：
|
||||
1. 准确性：回答是否与参考答案的核心信息一致
|
||||
2. 完整性：回答是否覆盖了参考答案的关键要点
|
||||
3. 相关性：回答是否直接回答了用户问题
|
||||
4. 流畅性：回答是否通顺、易于理解
|
||||
|
||||
请以JSON格式返回评分：
|
||||
{{"accuracy": X, "completeness": X, "relevance": X, "fluency": X, "overall": X}}
|
||||
|
||||
只返回JSON，不要其他内容。"""
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=self._llm_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
result_text = response.choices[0].message.content.strip()
|
||||
|
||||
# 尝试解析JSON
|
||||
import re
|
||||
json_match = re.search(r'\{[^}]+\}', result_text)
|
||||
if json_match:
|
||||
scores = json.loads(json_match.group())
|
||||
overall = scores.get('overall', 0)
|
||||
return min(1.0, max(0.0, overall / 10.0))
|
||||
else:
|
||||
# 尝试从文本中提取数字
|
||||
numbers = re.findall(r'\d+', result_text)
|
||||
if numbers:
|
||||
return min(1.0, max(0.0, int(numbers[-1]) / 10.0))
|
||||
return 0.5
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM评分失败: {e}")
|
||||
return 0.0
|
||||
|
||||
def generate_answer(self, query: str, top_k: int = 5, collections: list = None) -> str:
|
||||
"""
|
||||
使用RAG引擎生成答案
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
top_k: 检索数量
|
||||
collections: 目标向量库列表
|
||||
|
||||
Returns:
|
||||
生成的答案
|
||||
"""
|
||||
engine = self._get_engine()
|
||||
|
||||
try:
|
||||
# 先检索
|
||||
results = engine.search_knowledge(
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
collections=collections
|
||||
)
|
||||
|
||||
# 提取文档内容
|
||||
if isinstance(results, dict):
|
||||
documents = results.get('documents', [])
|
||||
if documents and isinstance(documents[0], list):
|
||||
documents = documents[0]
|
||||
else:
|
||||
documents = []
|
||||
|
||||
if not documents:
|
||||
return "抱歉，未找到相关信息。"
|
||||
|
||||
# 简单拼接作为答案（实际应该用LLM生成）
|
||||
# 这里为了评测，我们用检索到的内容拼接
|
||||
context = "\n\n".join(documents[:3])
|
||||
|
||||
# 使用LLM生成答案
|
||||
client = self._get_llm_client()
|
||||
if client:
|
||||
prompt = f"""基于以下检索到的信息回答用户问题。请简洁准确地回答。
|
||||
|
||||
【用户问题】
|
||||
{query}
|
||||
|
||||
【检索到的信息】
|
||||
{context}
|
||||
|
||||
请直接回答问题，不要重复问题本身："""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self._llm_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3,
|
||||
max_tokens=500
|
||||
)
|
||||
return response.choices[0].message.content.strip()
|
||||
else:
|
||||
# 没有LLM时，直接返回最相关的文档片段
|
||||
return documents[0] if documents else "无法生成答案"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成答案失败: {e}")
|
||||
return f"生成答案时出错: {str(e)}"
|
||||
|
||||
def evaluate_query(
|
||||
self,
|
||||
query: str,
|
||||
reference_answer: str,
|
||||
top_k: int = 5,
|
||||
collections: list = None,
|
||||
use_llm: bool = True
|
||||
) -> dict:
|
||||
"""
|
||||
评估单个查询的答案质量
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
reference_answer: 参考答案
|
||||
top_k: 检索数量
|
||||
collections: 目标向量库列表
|
||||
use_llm: 是否使用LLM评分
|
||||
|
||||
Returns:
|
||||
评测结果字典
|
||||
"""
|
||||
# 生成答案
|
||||
generated_answer = self.generate_answer(query, top_k=top_k, collections=collections)
|
||||
|
||||
# 计算各项指标
|
||||
metrics = {
|
||||
'rouge_l': self.rouge_l(generated_answer, reference_answer),
|
||||
'semantic_similarity': self.semantic_similarity(generated_answer, reference_answer),
|
||||
'generated_answer': generated_answer[:500], # 截断保存
|
||||
'reference_answer': reference_answer[:500]
|
||||
}
|
||||
|
||||
# LLM评分（可选，因为较慢）
|
||||
if use_llm:
|
||||
metrics['llm_score'] = self.llm_score(query, generated_answer, reference_answer)
|
||||
|
||||
return metrics
|
||||
|
||||
def evaluate_dataset(
|
||||
self,
|
||||
eval_dataset_path: str,
|
||||
top_k: int = 5,
|
||||
collections: list = None,
|
||||
use_llm: bool = True,
|
||||
sample_size: int = None
|
||||
) -> dict:
|
||||
"""
|
||||
评估整个数据集
|
||||
|
||||
Args:
|
||||
eval_dataset_path: 评测数据集路径
|
||||
top_k: 检索数量
|
||||
collections: 目标向量库列表
|
||||
use_llm: 是否使用LLM评分
|
||||
sample_size: 采样数量（用于快速测试）
|
||||
|
||||
Returns:
|
||||
汇总评测结果
|
||||
"""
|
||||
# 加载数据集
|
||||
with open(eval_dataset_path, 'r', encoding='utf-8') as f:
|
||||
dataset = json.load(f)
|
||||
|
||||
queries = dataset.get('queries', [])
|
||||
|
||||
# 采样（如果指定）
|
||||
if sample_size and sample_size < len(queries):
|
||||
import random
|
||||
queries = random.sample(queries, sample_size)
|
||||
|
||||
# 存储每个查询的结果
|
||||
all_metrics = {
|
||||
'rouge_l': [],
|
||||
'semantic_similarity': [],
|
||||
'llm_score': []
|
||||
}
|
||||
|
||||
# 按查询类型分组
|
||||
by_type = {}
|
||||
# 按难度分组
|
||||
by_difficulty = {}
|
||||
|
||||
logger.info(f"开始答案层评测，共 {len(queries)} 条查询...")
|
||||
|
||||
for i, q in enumerate(queries, 1):
|
||||
query_text = q['query']
|
||||
query_type = q.get('query_type', 'unknown')
|
||||
difficulty = q.get('difficulty', 'medium')
|
||||
reference_answer = q.get('reference_answer', '')
|
||||
|
||||
if not reference_answer:
|
||||
logger.warning(f"查询 {q['id']} 没有参考答案，跳过")
|
||||
continue
|
||||
|
||||
# 执行评测
|
||||
metrics = self.evaluate_query(
|
||||
query=query_text,
|
||||
reference_answer=reference_answer,
|
||||
top_k=top_k,
|
||||
collections=collections,
|
||||
use_llm=use_llm
|
||||
)
|
||||
|
||||
# 收集结果
|
||||
all_metrics['rouge_l'].append(metrics['rouge_l'])
|
||||
all_metrics['semantic_similarity'].append(metrics['semantic_similarity'])
|
||||
if 'llm_score' in metrics:
|
||||
all_metrics['llm_score'].append(metrics['llm_score'])
|
||||
|
||||
# 按类型分组
|
||||
if query_type not in by_type:
|
||||
by_type[query_type] = {k: [] for k in all_metrics}
|
||||
by_type[query_type]['rouge_l'].append(metrics['rouge_l'])
|
||||
by_type[query_type]['semantic_similarity'].append(metrics['semantic_similarity'])
|
||||
if 'llm_score' in metrics:
|
||||
by_type[query_type]['llm_score'].append(metrics['llm_score'])
|
||||
|
||||
# 按难度分组
|
||||
if difficulty not in by_difficulty:
|
||||
by_difficulty[difficulty] = {k: [] for k in all_metrics}
|
||||
by_difficulty[difficulty]['rouge_l'].append(metrics['rouge_l'])
|
||||
by_difficulty[difficulty]['semantic_similarity'].append(metrics['semantic_similarity'])
|
||||
if 'llm_score' in metrics:
|
||||
by_difficulty[difficulty]['llm_score'].append(metrics['llm_score'])
|
||||
|
||||
# 打印进度
|
||||
llm_str = f", LLM={metrics.get('llm_score', 0):.2f}" if 'llm_score' in metrics else ""
|
||||
logger.info(f" [{i}/{len(queries)}] {q['id']}: ROUGE-L={metrics['rouge_l']:.2f}, SemSim={metrics['semantic_similarity']:.2f}{llm_str}")
|
||||
|
||||
# 计算平均值
|
||||
results = {
|
||||
'overall': {
|
||||
key: np.mean(values) if values else 0.0
|
||||
for key, values in all_metrics.items()
|
||||
},
|
||||
'by_type': {
|
||||
qtype: {key: np.mean(vals) if vals else 0.0 for key, vals in metrics.items()}
|
||||
for qtype, metrics in by_type.items()
|
||||
},
|
||||
'by_difficulty': {
|
||||
diff: {key: np.mean(vals) if vals else 0.0 for key, vals in metrics.items()}
|
||||
for diff, metrics in by_difficulty.items()
|
||||
},
|
||||
'total_queries': len(queries),
|
||||
'evaluated_queries': len(all_metrics['rouge_l'])
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def print_results(self, results: dict):
|
||||
"""打印评测结果"""
|
||||
print("\n" + "=" * 60)
|
||||
print(" RAG 答案层评测结果")
|
||||
print("=" * 60)
|
||||
|
||||
# 整体结果
|
||||
print("\n【整体指标】")
|
||||
overall = results.get('overall', {})
|
||||
print(f" ROUGE-L: {overall.get('rouge_l', 0):.4f}")
|
||||
print(f" Semantic Sim: {overall.get('semantic_similarity', 0):.4f}")
|
||||
if overall.get('llm_score'):
|
||||
print(f" LLM Score: {overall.get('llm_score', 0):.4f}")
|
||||
|
||||
# 按查询类型
|
||||
print("\n【按查询类型】")
|
||||
by_type = results.get('by_type', {})
|
||||
for qtype, metrics in sorted(by_type.items()):
|
||||
print(f" {qtype}:")
|
||||
print(f" ROUGE-L: {metrics.get('rouge_l', 0):.4f}")
|
||||
print(f" SemSim: {metrics.get('semantic_similarity', 0):.4f}")
|
||||
|
||||
# 按难度
|
||||
print("\n【按难度】")
|
||||
by_difficulty = results.get('by_difficulty', {})
|
||||
for diff, metrics in sorted(by_difficulty.items()):
|
||||
print(f" {diff}:")
|
||||
print(f" ROUGE-L: {metrics.get('rouge_l', 0):.4f}")
|
||||
print(f" SemSim: {metrics.get('semantic_similarity', 0):.4f}")
|
||||
|
||||
# 统计信息
|
||||
print("\n【统计信息】")
|
||||
print(f" 总查询数: {results.get('total_queries', 0)}")
|
||||
print(f" 已评测数: {results.get('evaluated_queries', 0)}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='RAG 答案层评测脚本')
|
||||
parser.add_argument(
|
||||
'--eval_dataset',
|
||||
type=str,
|
||||
default='data/eval_dataset.json',
|
||||
help='评测数据集路径'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--topk',
|
||||
type=int,
|
||||
default=5,
|
||||
help='检索返回数量 (默认: 5)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--collections',
|
||||
type=str,
|
||||
nargs='+',
|
||||
default=None,
|
||||
help='目标向量库列表'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--no_llm',
|
||||
action='store_true',
|
||||
help='跳过LLM评分（更快但指标较少）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--sample',
|
||||
type=int,
|
||||
default=None,
|
||||
help='采样数量（用于快速测试）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
type=str,
|
||||
default=None,
|
||||
help='结果输出文件路径 (JSON格式)'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 检查数据集文件
|
||||
eval_path = PROJECT_ROOT / args.eval_dataset
|
||||
if not eval_path.exists():
|
||||
logger.error(f"评测数据集不存在: {eval_path}")
|
||||
sys.exit(1)
|
||||
|
||||
# 创建评测器
|
||||
logger.info("初始化答案层评测器...")
|
||||
evaluator = AnswerEvaluator()
|
||||
|
||||
# 执行评测
|
||||
start_time = time.time()
|
||||
results = evaluator.evaluate_dataset(
|
||||
eval_dataset_path=str(eval_path),
|
||||
top_k=args.topk,
|
||||
collections=args.collections,
|
||||
use_llm=not args.no_llm,
|
||||
sample_size=args.sample
|
||||
)
|
||||
elapsed_time = time.time() - start_time
|
||||
|
||||
# 打印结果
|
||||
evaluator.print_results(results)
|
||||
print(f"\n评测耗时: {elapsed_time:.2f} 秒")
|
||||
|
||||
# 保存结果
|
||||
if args.output:
|
||||
output_path = PROJECT_ROOT / args.output
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(results, f, ensure_ascii=False, indent=2)
|
||||
logger.info(f"结果已保存到: {output_path}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user