fix(bm25): 修复 BM25 索引覆盖 bug + 综合评测集 v2 + 检索增强
核心修复: - knowledge/base.py: BM25Index.add_documents 从覆盖改为追加+去重， 修复只有最后上传文件的 chunks 保留在 BM25 中的严重 bug (影响: 2.docx/3.docx/PDF 的 755 个 chunk 在 BM25 中完全缺失) 检索增强 (延续上次会话): - core/engine.py: section cluster boost + lexical match exemption - api/chat_routes.py: lexical/cluster rescue 层 + SSE 事件 - core/mmr.py: MMR 去重改进 评测体系: - tests/eval_dataset_v2.json: 62 题综合评测集 (9 种题型×4 文档) - scripts/eval_e2e.py: 推理模型 LLM 评分兼容 + 新数据集格式支持 - scripts/validate_eval_dataset.py: 数据集验证工具 其他: - parsers/mineru_parser.py: 解析器改进
This commit is contained in:
lacerate551
@@ -54,7 +54,7 @@ def setup_file_logging(log_path: str):
|
||||
|
||||
# ───────────── 配置 ─────────────
|
||||
RAG_API_URL = "http://127.0.0.1:5001/rag"
|
||||
DEFAULT_DATASET = "data/eval/eval_dataset.json"
|
||||
DEFAULT_DATASET = "tests/eval_dataset_v2.json"
|
||||
RESULTS_DIR = "data/eval_results"
|
||||
REQUEST_TIMEOUT = 120 # 秒
|
||||
REQUEST_INTERVAL = 1.5 # 请求间隔（秒），避免过载
|
||||
@@ -79,7 +79,8 @@ def call_rag_sse(question: str, collections: list = None, api_url: str = None) -
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream"
|
||||
"Accept": "text/event-stream",
|
||||
"Authorization": "Bearer mock-token-admin"
|
||||
}
|
||||
# chat_history 在生产模式下是必填字段
|
||||
payload = {"message": question, "chat_history": []}
|
||||
@@ -185,28 +186,98 @@ def llm_quality_score(query: str, answer: str, reference: str) -> dict:
|
||||
3. 相关性(relevance)：回答是否直接针对问题，没有跑题或冗余
|
||||
4. 流畅性(fluency)：回答是否通顺、结构清晰、易于理解
|
||||
|
||||
请严格按以下 JSON 格式返回，不要包含其他内容：
|
||||
{{"accuracy": <分数>, "completeness": <分数>, "relevance": <分数>, "fluency": <分数>, "overall": <总分>}}"""
|
||||
【输出要求】
|
||||
只输出一个纯 JSON 对象，不要包含任何其他文字、解释或 markdown 格式：
|
||||
{{"accuracy": <整数>, "completeness": <整数>, "relevance": <整数>, "fluency": <整数>, "overall": <整数>}}"""
|
||||
|
||||
def _extract_json(text: str) -> dict | None:
|
||||
"""从文本中提取 JSON，支持嵌套大括号"""
|
||||
if not text:
|
||||
return None
|
||||
# 1. 移除推理模型的思考标签及其内容
|
||||
text = re.sub(r'<think>[\s\S]*?</think>', '', text).strip()
|
||||
# 2. 尝试直接解析整个文本
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# 3. 贪婪匹配最大的 {...} 块（支持嵌套）
|
||||
# 从最后一个 } 往前找匹配的 {
|
||||
brace_depth = 0
|
||||
start = -1
|
||||
end = -1
|
||||
for i in range(len(text) - 1, -1, -1):
|
||||
if text[i] == '}':
|
||||
if brace_depth == 0:
|
||||
end = i
|
||||
brace_depth += 1
|
||||
elif text[i] == '{':
|
||||
brace_depth -= 1
|
||||
if brace_depth == 0:
|
||||
start = i
|
||||
break
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
return json.loads(text[start:end + 1])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# 4. 回退：简单单层 {...} 匹配
|
||||
m = re.search(r'\{[^{}]+\}', text)
|
||||
if m:
|
||||
try:
|
||||
return json.loads(m.group())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return None
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=DASHSCOPE_MODEL,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=300
|
||||
)
|
||||
text = response.choices[0].message.content.strip()
|
||||
# 构建请求参数
|
||||
request_params = {
|
||||
"model": DASHSCOPE_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a JSON-only evaluator. Output ONLY a valid JSON object, nothing else."},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 2000,
|
||||
}
|
||||
# 尝试关闭推理模型的思考输出（部分 API 支持）
|
||||
try:
|
||||
request_params["extra_body"] = {"enable_thinking": False}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取 JSON
|
||||
json_match = re.search(r'\{[^}]+\}', text)
|
||||
if json_match:
|
||||
scores = json.loads(json_match.group())
|
||||
response = client.chat.completions.create(**request_params)
|
||||
msg = response.choices[0].message
|
||||
|
||||
# 优先取 content，如果为空则尝试 reasoning_content 后的内容
|
||||
text = getattr(msg, 'content', '') or ''
|
||||
|
||||
# 如果 content 为空，尝试从 reasoning_content 中提取
|
||||
# （部分推理模型 API 将思考内容放在 reasoning_content，回答放在 content）
|
||||
if not text.strip():
|
||||
reasoning = getattr(msg, 'reasoning_content', '') or ''
|
||||
if reasoning:
|
||||
# 从思考内容末尾尝试提取 JSON
|
||||
text = reasoning
|
||||
|
||||
if not text.strip():
|
||||
logger.warning("LLM 返回内容为空")
|
||||
return {"overall": 0.0, "error": "empty_response"}
|
||||
|
||||
scores = _extract_json(text)
|
||||
if scores:
|
||||
# 归一化到 0-1
|
||||
for k in scores:
|
||||
scores[k] = round(min(10, max(0, scores[k])) / 10.0, 4)
|
||||
for k in list(scores.keys()):
|
||||
try:
|
||||
scores[k] = round(min(10, max(0, float(scores[k]))) / 10.0, 4)
|
||||
except (ValueError, TypeError):
|
||||
scores[k] = 0.0
|
||||
return scores
|
||||
else:
|
||||
logger.warning(f"LLM 返回内容无法解析 JSON: {text[:100]}")
|
||||
# 记录更多内容用于调试
|
||||
debug_text = text[:200].replace('\n', '\\n')
|
||||
logger.warning(f"LLM 返回内容无法解析 JSON: {debug_text}")
|
||||
return {"overall": 0.0, "error": "parse_failed"}
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM 评分异常: {e}")
|
||||
@@ -228,7 +299,7 @@ def evaluate_dataset(dataset_path: str, use_llm: bool = True, api_url: str = Non
|
||||
with open(dataset_path, 'r', encoding='utf-8') as f:
|
||||
dataset = json.load(f)
|
||||
|
||||
questions = dataset.get("questions", [])
|
||||
questions = dataset.get("queries", dataset.get("questions", []))
|
||||
total = len(questions)
|
||||
if total == 0:
|
||||
logger.error("数据集中没有问题")
|
||||
|
||||
109
scripts/validate_eval_dataset.py
Normal file
109
scripts/validate_eval_dataset.py
Normal file
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
验证 eval_dataset_v2.json 的完整性和格式正确性
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
DATASET_PATH = PROJECT_ROOT / "tests" / "eval_dataset_v2.json"
|
||||
|
||||
|
||||
def validate():
|
||||
errors = []
|
||||
warnings = []
|
||||
|
||||
with open(DATASET_PATH, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
queries = data.get("queries", [])
|
||||
print(f"[OK] Dataset loaded: {len(queries)} queries")
|
||||
|
||||
# 1. Check required fields
|
||||
required = {"id", "query", "query_type", "relevant_docs", "reference_answer", "expected_keywords", "difficulty"}
|
||||
for q in queries:
|
||||
missing = required - set(q.keys())
|
||||
if missing:
|
||||
errors.append(f" {q.get('id', '?')}: missing fields: {missing}")
|
||||
|
||||
# 2. Check unique IDs
|
||||
ids = [q["id"] for q in queries]
|
||||
dupes = [id for id, cnt in Counter(ids).items() if cnt > 1]
|
||||
if dupes:
|
||||
errors.append(f" Duplicate IDs: {dupes}")
|
||||
|
||||
# 3. Check query types
|
||||
valid_types = {"simple_fact", "enumeration", "definition", "comparison",
|
||||
"reasoning", "table_data", "cross_doc", "negative", "paraphrase"}
|
||||
type_counts = Counter(q["query_type"] for q in queries)
|
||||
invalid_types = set(type_counts.keys()) - valid_types
|
||||
if invalid_types:
|
||||
errors.append(f" Invalid query_types: {invalid_types}")
|
||||
|
||||
print(f"\n[INFO] Query type distribution:")
|
||||
for t, c in sorted(type_counts.items(), key=lambda x: -x[1]):
|
||||
marker = " [INVALID]" if t in (invalid_types or set()) else ""
|
||||
print(f" {t}: {c}{marker}")
|
||||
|
||||
# 4. Check difficulty distribution
|
||||
diff_counts = Counter(q["difficulty"] for q in queries)
|
||||
print(f"\n[INFO] Difficulty distribution:")
|
||||
for d in ["easy", "medium", "hard"]:
|
||||
print(f" {d}: {diff_counts.get(d, 0)}")
|
||||
|
||||
# 5. Check document coverage
|
||||
doc_counts = Counter()
|
||||
for q in queries:
|
||||
for doc in q["relevant_docs"]:
|
||||
doc_counts[doc] += 1
|
||||
if not any("negative" in q["query_type"] for q in queries):
|
||||
warnings.append(" No negative test cases")
|
||||
|
||||
print(f"\n[INFO] Document coverage:")
|
||||
for doc, cnt in sorted(doc_counts.items(), key=lambda x: -x[1]):
|
||||
print(f" {doc}: {cnt} queries")
|
||||
neg_count = sum(1 for q in queries if q["query_type"] == "negative")
|
||||
print(f" (negative/out-of-scope): {neg_count} queries")
|
||||
|
||||
# 6. Check expected_keywords
|
||||
empty_kw = [q["id"] for q in queries if not q.get("expected_keywords")]
|
||||
if empty_kw:
|
||||
warnings.append(f" Empty expected_keywords: {empty_kw}")
|
||||
|
||||
# 7. Check paraphrase references
|
||||
paraphrases = [q for q in queries if q["query_type"] == "paraphrase"]
|
||||
for p in paraphrases:
|
||||
ref = p.get("paraphrase_of")
|
||||
if ref and ref not in ids:
|
||||
errors.append(f" {p['id']}: paraphrase_of '{ref}' not found")
|
||||
|
||||
# 8. Check reference_answer length
|
||||
short_refs = [q["id"] for q in queries if len(q.get("reference_answer", "")) < 10]
|
||||
if short_refs:
|
||||
warnings.append(f" Very short reference_answer: {short_refs}")
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
if errors:
|
||||
print(f"[ERROR] {len(errors)} error(s):")
|
||||
for e in errors:
|
||||
print(e)
|
||||
if warnings:
|
||||
print(f"[WARN] {len(warnings)} warning(s):")
|
||||
for w in warnings:
|
||||
print(w)
|
||||
if not errors and not warnings:
|
||||
print("[PASS] Dataset validation passed!")
|
||||
print(f"{'='*60}")
|
||||
|
||||
return len(errors) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.platform == 'win32':
|
||||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||||
ok = validate()
|
||||
sys.exit(0 if ok else 1)
|
||||
Reference in New Issue
Block a user