jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bm25): 修复 BM25 索引覆盖 bug + 综合评测集 v2 + 检索增强

Browse Source 
核心修复:
- knowledge/base.py: BM25Index.add_documents 从覆盖改为追加+去重，
  修复只有最后上传文件的 chunks 保留在 BM25 中的严重 bug
  (影响: 2.docx/3.docx/PDF 的 755 个 chunk 在 BM25 中完全缺失)

检索增强 (延续上次会话):
- core/engine.py: section cluster boost + lexical match exemption
- api/chat_routes.py: lexical/cluster rescue 层 + SSE 事件
- core/mmr.py: MMR 去重改进

评测体系:
- tests/eval_dataset_v2.json: 62 题综合评测集 (9 种题型×4 文档)
- scripts/eval_e2e.py: 推理模型 LLM 评分兼容 + 新数据集格式支持
- scripts/validate_eval_dataset.py: 数据集验证工具

其他:
- parsers/mineru_parser.py: 解析器改进
This commit is contained in:
lacerate551
2026-06-17 17:31:43 +08:00
parent edaef7ad60
commit 8c92657490
8 changed files with 1322 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

228
api/chat_routes.py
View File

@@ -493,6 +493,198 @@ def _section_similarity(section_a: str, section_b: str) -> float:
return overlap / union if union > 0 else 0.0
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
min_score: float) -> List[Dict]:
"""
词法匹配救援：当切片文本精确包含查询的核心关键词但 CrossEncoder 评分很低时，
将其分数提升至保底值，并同时救援同 source 下 chunk_index 相邻的切片。
适用场景：
1. 某个 section 只有一个正确切片（无法触发聚类救援）
2. 枚举类问题的 header 切片被词法匹配救援后，其后续子条目也应被一并保留
Args:
contexts: 全部上下文切片
retrieval_query: 检索查询
min_score: 最低分数阈值
Returns:
修改后的 contexts
"""
if not contexts or not retrieval_query:
return contexts
import re
from config import CLUSTER_RESCUE_FLOOR
# 清理查询：去除 markdown 格式和标点
clean_query = re.sub(r'\*+|#+|`', '', retrieval_query)
clean_query = re.sub(r'[？?！!。，,、；;：:"""\'\s]+', ' ', clean_query).strip()
if len(clean_query) < 2:
return contexts
# 提取查询中的有意义 bigram（连续两字组）
query_bigrams = set()
for i in range(len(clean_query) - 1):
w = clean_query[i:i+2].strip()
if len(w) == 2:
query_bigrams.add(w)
if not query_bigrams:
return contexts
# Phase 1: 词法匹配救援——找到高分匹配的切片
rescued_seeds = [] # [(source, chunk_index)]
for ctx in contexts:
if ctx.get('score', 0) >= min_score:
continue
doc = ctx.get('doc', '') or ''
meta = ctx.get('meta', {})
section = meta.get('section', '') or meta.get('section_path', '')
combined = doc + ' ' + section
matched = sum(1 for w in query_bigrams if w in combined)
match_ratio = matched / len(query_bigrams)
if match_ratio > 0.35:
ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
source = meta.get('source', '')
chunk_index = meta.get('chunk_index')
if source and chunk_index is not None:
try:
rescued_seeds.append((source, int(chunk_index)))
except (ValueError, TypeError):
pass
# Phase 2: 邻居救援——对每个被词法匹配救援的种子，
# 同时救援同 source 下 chunk_index 后续相邻的切片（枚举子条目）
if rescued_seeds:
for source, seed_idx in rescued_seeds:
for ctx in contexts:
if ctx.get('score', 0) >= min_score:
continue
meta = ctx.get('meta', {})
if meta.get('source') != source:
continue
try:
n_idx = int(meta.get('chunk_index', -1))
except (ValueError, TypeError):
continue
# 救援种子后续 8 个相邻切片（覆盖大多数枚举/条款模式）
if seed_idx < n_idx <= seed_idx + 8:
ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
return contexts
def _normalize_section_for_rescue(section_path: str, levels: int = 2) -> str:
"""归一化 section_path：取前 N 级路径用于分组"""
if not section_path:
return ''
parts = [p.strip() for p in section_path.split('>')]
return ' > '.join(parts[:levels])
def _rescue_section_cluster(contexts: List[Dict], retrieval_query: str,
min_score: float) -> List[Dict]:
"""
路由层章节聚类救援：当某个 section 有多个候选切片但全部低于 min_score 时，
给该 section 的切片分配保底分数，使其通过后续的 min_score 过滤。
作为引擎层 _section_cluster_boost 的二次安全网，防止极端情况下所有正确切片被过滤。
Args:
contexts: engine 返回的全部上下文切片（每个含 doc, meta, score）
retrieval_query: 改写后的检索查询
min_score: 当前的最低分数阈值
Returns:
修改后的 contexts（部分切片 score 被提升至保底分数）
"""
if not contexts:
return contexts
from config import (
CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, CLUSTER_RESCUE_FLOOR,
CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION,
CLUSTER_SECTION_PREFIX_LEVELS,
)
# 1. 按 (source, normalized_section) 分组
from collections import defaultdict
section_groups = defaultdict(list) # key → [index_in_contexts]
for i, ctx in enumerate(contexts):
meta = ctx.get('meta', {})
source = meta.get('source', '')
section_path = meta.get('section', '') or meta.get('section_path', '')
norm_section = _normalize_section_for_rescue(section_path, CLUSTER_SECTION_PREFIX_LEVELS)
if not source or not norm_section:
continue
key = (source, norm_section)
section_groups[key].append(i)
# 2. 检测"全灭 section"并计算聚类强度
rescue_candidates = [] # (strength, key, member_indices)
for key, indices in section_groups.items():
if len(indices) < CLUSTER_MIN_MEMBERS:
continue
# 检查是否所有成员都低于 min_score（"全灭"）
scores = [contexts[i].get('score', 0) for i in indices]
if any(s >= min_score for s in scores):
continue # 已有成员通过阈值，无需救援
# 计算聚类强度
chunk_types = set(contexts[i].get('meta', {}).get('chunk_type', 'text') for i in indices)
type_diversity = len(chunk_types)
if type_diversity < CLUSTER_MIN_TYPES:
continue # 类型不够多样，可能是噪音
# 查询与 section_path 的字符重叠率
source, norm_section = key
query_chars = set(retrieval_query)
section_chars = set(norm_section)
overlap = len(query_chars & section_chars) / max(len(query_chars), 1)
# 聚类强度 = 成员数 × 类型多样性 × (1 + 查询匹配度)
strength = len(indices) * type_diversity * (1.0 + overlap)
rescue_candidates.append((strength, key, indices))
# 3. 按强度降序，救援 top-N section
rescue_candidates.sort(key=lambda x: x[0], reverse=True)
rescued_sections = []
total_rescued = 0
for strength, key, indices in rescue_candidates[:CLUSTER_MAX_SECTIONS]:
# 对组内切片分配保底分数（仅低于 min_score 的）
rescue_count = 0
for idx in indices:
if rescue_count >= CLUSTER_MAX_RESCUE_PER_SECTION:
break
ctx = contexts[idx]
if ctx.get('score', 0) < min_score:
ctx['score'] = CLUSTER_RESCUE_FLOOR
rescue_count += 1
total_rescued += 1
if rescue_count > 0:
source, norm_section = key
rescued_sections.append({
'source': source,
'section': norm_section,
'members': len(indices),
'rescued': rescue_count,
'strength': round(strength, 2)
})
return contexts
def _rescue_table_chunks(contexts: List[Dict], context_text: str,
retrieval_query: str, max_rescue_chars: int = 3000) -> str:
"""
@@ -1731,7 +1923,10 @@ def rag():
DIRECT_CONTEXT_MAX_CHARS,
RERANK_CONTEXT_MIN_SCORE,
CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT,
CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD
CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD,
SECTION_CLUSTER_RESCUE_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
CLUSTER_RESCUE_FLOOR, CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION,
CLUSTER_SECTION_PREFIX_LEVELS
)
data = request.json or {}
@@ -1973,6 +2168,11 @@ def rag():
debug_info = search_result.get('_debug', {})
yield f"data: {json.dumps({'type': 'retrieval_debug', 'data': {'steps': debug_info.get('steps', []), 'collections_searched': collections, 'total_candidates': len(search_result.get('ids', [[]])[0])}}, ensure_ascii=False)}\n\n"
# 提取章节聚类提升事件（引擎层）单独发送
for step in debug_info.get('steps', []):
if step.get('name') == 'section_cluster_boost':
yield f"data: {json.dumps({'type': 'section_cluster_boost', 'data': step}, ensure_ascii=False)}\n\n"
# 提取上下文
contexts = []
sources = []
@@ -2241,6 +2441,32 @@ def rag():
# 4. 构建 prompt（Phase 6：LLM 图片感知）
# Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争
# 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时，
# 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景）
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
# 章节聚类救援（路由层安全网）：在 min_score 过滤前，
# 检测"全灭 section"并分配保底分数
_rescue_debug = None
if SECTION_CLUSTER_RESCUE_ENABLED:
_before_scores = {id(ctx): ctx.get('score', 0) for ctx in contexts}
contexts = _rescue_section_cluster(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
# 统计救援结果
_rescued = [ctx for ctx in contexts if id(ctx) in _before_scores
and ctx.get('score', 0) > _before_scores[id(ctx)]]
if _rescued and IS_DEV:
_rescue_sections = set()
for ctx in _rescued:
meta = ctx.get('meta', {})
sp = meta.get('section', '') or meta.get('section_path', '')
_rescue_sections.add(_normalize_section_for_rescue(sp, CLUSTER_SECTION_PREFIX_LEVELS))
_rescue_debug = {
'rescued_count': len(_rescued),
'rescued_sections': list(_rescue_sections)
}
yield f"data: {json.dumps({'type': 'section_cluster_rescue', 'data': _rescue_debug}, ensure_ascii=False)}\n\n"
text_contexts = _order_text_contexts_for_prompt(contexts, retrieval_query, MAX_CONTEXT_CHUNKS,
min_score=RERANK_CONTEXT_MIN_SCORE)
# Phase 2：按字符预算构建上下文

171
core/engine.py
View File

@@ -77,6 +77,10 @@ try:
CONTEXT_EXPANSION_MAX_CHUNKS, ENUM_QUERY_DISABLE_TOPK_SHRINK, ENUM_QUERY_MMR_LAMBDA,
# Phase 3 扩展精细化
EXPANSION_SCORE_THRESHOLD, MAX_EXPANDED_NEIGHBORS,
# 章节聚类救援
SECTION_CLUSTER_BOOST_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
CLUSTER_SEED_FLOOR, CLUSTER_MAX_BOOST_PER_SECTION, CLUSTER_MAX_SECTIONS,
CLUSTER_SECTION_PREFIX_LEVELS,
# 上下文与生成
LLM_TEMPERATURE, LLM_MAX_TOKENS, RECALL_MULTIPLIER,
# FAQ 与黑名单
@@ -102,10 +106,18 @@ except ImportError:
MMR_TOP_K = 30
CONTEXT_EXPANSION_ENABLED = True
CONTEXT_EXPANSION_BEFORE = 1
CONTEXT_EXPANSION_AFTER = 5
CONTEXT_EXPANSION_AFTER = 8
CONTEXT_EXPANSION_MAX_CHUNKS = 24
EXPANSION_SCORE_THRESHOLD = 0.3
MAX_EXPANDED_NEIGHBORS = 4
MAX_EXPANDED_NEIGHBORS = 8
# 章节聚类救援默认值
SECTION_CLUSTER_BOOST_ENABLED = True
CLUSTER_MIN_MEMBERS = 3
CLUSTER_MIN_TYPES = 2
CLUSTER_SEED_FLOOR = 0.35
CLUSTER_MAX_BOOST_PER_SECTION = 8
CLUSTER_MAX_SECTIONS = 3
CLUSTER_SECTION_PREFIX_LEVELS = 1
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
ENUM_QUERY_MMR_LAMBDA = 0.85
DYNAMIC_RRF_ENABLED = True
@@ -732,11 +744,19 @@ class RAGEngine:
# 时间衰减（Time Decay）
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})
@@ -1160,14 +1180,139 @@ class RAGEngine:
return None
return self.collection
@staticmethod
def _normalize_section_path(section_path: str, levels: int = None) -> str:
"""归一化 section_path：取前 N 级路径，容忍 MinerU 标题检测误差。
例如: "第三章 吸烟场所的功能设置 > 第三条 文明吸烟..." → "第三章 吸烟场所的功能设置"
"""
if not section_path:
return ''
if levels is None:
levels = CLUSTER_SECTION_PREFIX_LEVELS
parts = [p.strip() for p in section_path.split('>')]
return ' > '.join(parts[:levels])
def _section_cluster_boost(self, results: dict, query: str = '') -> dict:
"""章节聚类提升：当同一 section 下多个切片（text+table）同时出现在候选集中，
即使单个切片 CrossEncoder 分数很低，也将整组提升到种子阈值。
核心洞察：单个低分切片不可信，但同一 section 多个切片同时出现是强信号。
提升后的切片可以作为 _expand_contiguous_chunks 的种子，触发邻居扩展。
Args:
results: rerank 后的检索结果
query: 用户查询（用于后续扩展）
Returns:
修改后的 results（distances 被调整，meta 中标记 _cluster_boosted）
"""
if not results.get('ids') or not results['ids'][0]:
return results
ids = results['ids'][0]
metas = results.get('metadatas', [[]])[0]
distances = results.get('distances', [[]])[0] if results.get('distances') else None
if not distances:
return results
# 1. 按 (source, normalized_section) 分组
from collections import defaultdict
section_groups = defaultdict(list) # key → [(index, meta, dist)]
for i, (meta, dist) in enumerate(zip(metas, distances)):
source = meta.get('source', '')
section_path = meta.get('section', '') or meta.get('section_path', '')
norm_section = self._normalize_section_path(section_path)
if not source or not norm_section:
continue
key = (source, norm_section)
section_groups[key].append((i, meta, dist))
# 2. 检测聚类信号并提升
boost_target_dist = 1.0 - CLUSTER_SEED_FLOOR # score=0.35 → dist=0.65
boosted_sections = []
total_boosted = 0
# 按组成员数降序排列，优先处理最大聚类
sorted_groups = sorted(section_groups.items(), key=lambda x: len(x[1]), reverse=True)
for (source, norm_section), members in sorted_groups:
if len(boosted_sections) >= CLUSTER_MAX_SECTIONS:
break
# 聚类信号检测：成员数 >= 阈值 且 类型多样性 >= 阈值
chunk_types = set(m[1].get('chunk_type', 'text') for m in members)
if len(members) < CLUSTER_MIN_MEMBERS or len(chunk_types) < CLUSTER_MIN_TYPES:
continue
# 提升组内切片分数（仅提升低于阈值的）
boost_count = 0
for idx, meta, dist in members:
if boost_count >= CLUSTER_MAX_BOOST_PER_SECTION:
break
# 只提升分数低于种子阈值的切片（高分切片不需要）
if dist > boost_target_dist:
distances[idx] = boost_target_dist
meta['_cluster_boosted'] = True
boost_count += 1
total_boosted += 1
if boost_count > 0:
boosted_sections.append({
'source': source,
'section': norm_section,
'members': len(members),
'types': list(chunk_types),
'boosted': boost_count
})
# 3. 写 debug 信息
if boosted_sections:
debug_info = results.get('_debug', {})
if 'steps' not in debug_info:
debug_info['steps'] = []
debug_info['steps'].append({
'name': 'section_cluster_boost',
'sections': boosted_sections,
'total_boosted': total_boosted
})
results['_debug'] = debug_info
logger.info(f"[章节聚类提升] 提升 {total_boosted} 个切片，"
f"涉及 {len(boosted_sections)} 个 section: "
f"{[s['section'][:30] for s in boosted_sections]}")
return results
@staticmethod
def _chunk_lexical_score(chunk_text: str, query: str) -> float:
"""计算切片文本与查询的词法重叠度（bigram 命中率），用于辅助种子资格判定。"""
if not chunk_text or not query:
return 0.0
import re
clean_q = re.sub(r'[？?！!。，,、；;：:"""\'\s*#`]+', ' ', query).strip()
if len(clean_q) < 2:
return 0.0
bigrams = set()
for i in range(len(clean_q) - 1):
w = clean_q[i:i+2].strip()
if len(w) == 2:
bigrams.add(w)
if not bigrams:
return 0.0
matched = sum(1 for w in bigrams if w in chunk_text)
return matched / len(bigrams)
def _expand_contiguous_chunks(self, results: dict, top_k: int = None,
min_score: float = 0.0) -> dict:
min_score: float = 0.0, query: str = '') -> dict:
"""Add same-source same-section neighbor text chunks around strong hits.
Args:
results: 检索结果
top_k: 最大切片数
min_score: Phase 3 最低分数阈值，仅对 Rerank 分数高于此值的种子扩展
query: 查询文本，用于词法匹配辅助种子资格判定
"""
if not CONTEXT_EXPANSION_ENABLED:
return results
@@ -1197,7 +1342,7 @@ class RAGEngine:
seeds = [
(doc_id, doc, meta, dist)
for doc_id, doc, meta, dist in items[:base_limit]
if meta.get('chunk_type', 'text') == 'text'
if (meta.get('chunk_type', 'text') == 'text' or meta.get('_cluster_boosted'))
and meta.get('source')
and self._to_int(meta.get('chunk_index')) is not None
]
@@ -1208,8 +1353,12 @@ class RAGEngine:
break
# Phase 3：跳过分数低于阈值的种子（仅当 min_score > 0 时生效）
# 词法匹配豁免：CrossEncoder 低分但关键词重叠度高时仍允许作为种子
if min_score > 0 and seed_dist < min_score:
continue
if query and self._chunk_lexical_score(_seed_doc, query) > 0.3:
pass # 词法匹配度高，允许作为种子
else:
continue
source = seed_meta.get('source')
section = seed_meta.get('section', '') or seed_meta.get('section_path', '')
@@ -1627,11 +1776,19 @@ class RAGEngine:
# 时间衰减
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
if _debug is not None:
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})

81
core/mmr.py
View File

@@ -108,22 +108,44 @@ def mmr_rerank(
return selected
def _tokenize_words(text: str) -> set:
"""
使用 jieba 分词并过滤噪声，返回有意义的词集合。
过滤规则：
- 去除单字符词（如 "的", "了", "在"）—— 这些是停用词，对区分文档无意义
- 去除纯数字 / 纯标点
- 保留 2 字及以上的实词
"""
import jieba
words = set()
for w in jieba.cut(text):
w = w.strip()
if len(w) >= 2 and not w.isdigit():
words.add(w)
return words
def mmr_filter_by_content(
candidates: List[Dict],
top_k: int = 30,
similarity_threshold: float = 0.9
similarity_threshold: float = 0.85
) -> List[Dict]:
"""
基于内容相似度的去重（简化版，不需要 embedding）
基于 jieba 词级 Jaccard 相似度的去重（不需要 embedding）
与旧版字符级 set(text) 的区别：
- 旧版：set("安全生产管理制度") → {'安','全','生','产',...}，中文文档间字符集合高度重叠
- 新版：jieba 分词 → {"安全生产", "管理制度", ...}，词级集合区分度高
适用于：
- 没有 embedding 的情况
- 快速去重场景
- MMR_USE_EMBEDDING=False 时的快速去重
- 避免 CPU 编码 100+ 文档的 50 秒开销
Args:
candidates: 候选文档列表
top_k: 返回数量
similarity_threshold: 相似度阈值，超过则视为重复
similarity_threshold: 相似度阈值，超过则视为重复（默认 0.85）
Returns:
去重后的候选文档列表
@@ -134,35 +156,42 @@ def mmr_filter_by_content(
if len(candidates) <= top_k:
return candidates
selected = []
remaining = candidates.copy()
# 预分词：对所有候选文档一次性分词，避免重复调用 jieba.cut
word_sets = []
for c in candidates:
content = c.get('content', c.get('document', ''))[:500]
word_sets.append(_tokenize_words(content))
while len(selected) < top_k and remaining:
current = remaining.pop(0)
selected_indices = []
for i in range(len(candidates)):
if len(selected_indices) >= top_k:
break
current_words = word_sets[i]
if not current_words:
# 空内容直接保留
selected_indices.append(i)
continue
# 检查是否与已选内容重复
is_duplicate = False
current_content = current.get('content', current.get('document', ''))[:200]
for j in selected_indices:
selected_words = word_sets[j]
if not selected_words:
continue
for s in selected:
s_content = s.get('content', s.get('document', ''))[:200]
intersection = len(current_words & selected_words)
union = len(current_words | selected_words)
similarity = intersection / union if union > 0 else 0
# 简单的 Jaccard 相似度
words1 = set(current_content)
words2 = set(s_content)
if words1 and words2:
intersection = len(words1 & words2)
union = len(words1 | words2)
similarity = intersection / union if union > 0 else 0
if similarity > similarity_threshold:
is_duplicate = True
break
if similarity > similarity_threshold:
is_duplicate = True
break
if not is_duplicate:
selected.append(current)
selected_indices.append(i)
return selected
return [candidates[i] for i in selected_indices]
# ==================== 测试 ====================

28
knowledge/base.py
View File

@@ -382,18 +382,34 @@ class BM25Index:
def add_documents(self, ids: List[str], documents: List[str], metadatas: List[dict]) -> None:
"""
添加文档到索引（会覆盖原有索引）
添加文档到索引（追加模式，自动去重）
如果 ID 已存在则更新对应文档，否则追加新文档。
添加后自动重建 BM25 索引。
Args:
ids: 文档 ID 列表
documents: 文档内容列表
metadatas: 文档元数据列表
"""
self.ids = ids
self.documents = documents
self.metadatas = metadatas
if documents:
tokenized = [self.tokenize(doc) for doc in documents]
# 建立已有 ID -> 索引位置 的映射，用于去重
existing_map = {doc_id: idx for idx, doc_id in enumerate(self.ids)}
for i, doc_id in enumerate(ids):
if doc_id in existing_map:
# 更新已有文档
pos = existing_map[doc_id]
self.documents[pos] = documents[i]
self.metadatas[pos] = metadatas[i]
else:
# 追加新文档
existing_map[doc_id] = len(self.ids)
self.ids.append(doc_id)
self.documents.append(documents[i])
self.metadatas.append(metadatas[i])
if self.documents:
tokenized = [self.tokenize(doc) for doc in self.documents]
self.bm25 = BM25Okapi(tokenized)
def search(self, query: str, top_k: int = 10) -> Tuple[List[str], List[str], List[dict], List[float]]:

54
parsers/mineru_parser.py
View File

@@ -137,8 +137,8 @@ def parse_with_mineru_online(
file_path: str,
api_token: str = None,
api_url: str = None,
model_version: str = "vlm",
timeout: int = 300
model_version: str = None,
timeout: int = None
) -> Dict[str, Any]:
"""
使用 MinerU 在线 API 解析文档
@@ -157,8 +157,8 @@ def parse_with_mineru_online(
file_path: 文档文件路径
api_token: API Token（默认从 config 读取）
api_url: API 地址
model_version: 模型版本 (vlm / pipeline / MinerU-HTML)
timeout: 请求超时（秒）
model_version: 模型版本 (vlm / pipeline / MinerU-HTML)，默认从 config 读取
timeout: 轮询超时（秒），默认从 config 读取
Returns:
解析结果（与 parse_with_mineru 格式相同）
@@ -167,10 +167,12 @@ def parse_with_mineru_online(
import time
import zipfile
import io
from config import MINERU_API_TOKEN, MINERU_API_URL
from config import MINERU_API_TOKEN, MINERU_API_URL, MINERU_MODEL_VERSION, MINERU_ONLINE_TIMEOUT
token = api_token or MINERU_API_TOKEN
url = api_url or MINERU_API_URL
model_version = model_version or MINERU_MODEL_VERSION
timeout = timeout or MINERU_ONLINE_TIMEOUT
if not token:
raise RuntimeError("MinerU 在线 API Token 未配置，请在 config.py 中设置 MINERU_API_TOKEN")
@@ -241,9 +243,15 @@ def parse_with_mineru_online(
result_resp.raise_for_status()
result = result_resp.json()
# 检查 API 层面的错误码，快速失败而非静默等到超时
api_code = result.get("code")
if api_code and api_code != 0:
api_msg = result.get("msg", "未知错误")
raise RuntimeError(f"MinerU API 错误 (code={api_code}): {api_msg}")
extract_results = result.get("data", {}).get("extract_result", [])
if not extract_results:
logger.debug(f"等待解析结果... ({waited}s)")
logger.debug(f"等待解析结果... ({waited}s/{max_wait}s)")
continue
# 取第一个文件的结果
@@ -276,11 +284,16 @@ def parse_with_mineru_online(
if progress:
extracted = progress.get("extracted_pages", 0)
total = progress.get("total_pages", 0)
logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s)")
logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s/{max_wait}s, model={model_version})")
else:
logger.debug(f"状态: {state}, 等待中... ({waited}s)")
logger.debug(f"状态: {state}, 等待中... ({waited}s/{max_wait}s)")
raise RuntimeError("MinerU 在线解析超时")
raise RuntimeError(
f"MinerU 在线解析超时 ({max_wait}s)"
f"，当前 model_version={model_version}"
f"，可尝试: 1) 设置 MINERU_MODEL_VERSION=pipeline 加速"
f" 2) 增大 MINERU_ONLINE_TIMEOUT"
)
except Exception as e:
raise RuntimeError(f"MinerU 在线 API 调用失败: {e}")
@@ -737,7 +750,7 @@ def parse_with_mineru(
lang: str = "ch",
enable_table: bool = True,
enable_formula: bool = True,
backend: str = "pipeline",
backend: str = None,
start_page: int = 0,
end_page: int = 99999
) -> Dict[str, Any]:
@@ -770,6 +783,14 @@ def parse_with_mineru(
if not file_path.exists():
raise FileNotFoundError(f"文件不存在: {file_path}")
# 从 config 读取默认 backend
if backend is None:
try:
from config import MINERU_LOCAL_BACKEND
backend = MINERU_LOCAL_BACKEND
except ImportError:
backend = 'pipeline'
# 检查文件大小
file_size = file_path.stat().st_size
if file_size > MAX_PDF_SIZE:
@@ -814,7 +835,6 @@ def parse_with_mineru(
cmd = [
str(mineru_exe),
"--",
"-p", str(file_path),
"-o", str(output_dir),
"-m", "auto",
@@ -859,7 +879,7 @@ def parse_with_mineru(
logger.error(f"MinerU 解析失败: {e}")
raise
finally:
清理临时目录
# 清理临时目录
if cleanup_output and os.path.exists(output_dir):
shutil.rmtree(output_dir, ignore_errors=True)
@@ -1688,7 +1708,7 @@ def parse_with_mineru_persistent(
lang: str = "ch",
enable_table: bool = True,
enable_formula: bool = True,
backend: str = "pipeline",
backend: str = None,
start_page: int = 0,
end_page: int = 99999,
cleanup_after_image_move: bool = True
@@ -1727,6 +1747,14 @@ def parse_with_mineru_persistent(
if not file_path.exists():
raise FileNotFoundError(f"文件不存在: {file_path}")
# 从 config 读取默认 backend
if backend is None:
try:
from config import MINERU_LOCAL_BACKEND
backend = MINERU_LOCAL_BACKEND
except ImportError:
backend = 'pipeline'
# 检查文件大小
file_size = file_path.stat().st_size
if file_size > MAX_PDF_SIZE:

109
scripts/eval_e2e.py
View File

@@ -54,7 +54,7 @@ def setup_file_logging(log_path: str):
# ───────────── 配置 ─────────────
RAG_API_URL = "http://127.0.0.1:5001/rag"
DEFAULT_DATASET = "data/eval/eval_dataset.json"
DEFAULT_DATASET = "tests/eval_dataset_v2.json"
RESULTS_DIR = "data/eval_results"
REQUEST_TIMEOUT = 120 # 秒
REQUEST_INTERVAL = 1.5 # 请求间隔（秒），避免过载
@@ -79,7 +79,8 @@ def call_rag_sse(question: str, collections: list = None, api_url: str = None) -
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream"
"Accept": "text/event-stream",
"Authorization": "Bearer mock-token-admin"
}
# chat_history 在生产模式下是必填字段
payload = {"message": question, "chat_history": []}
@@ -185,28 +186,98 @@ def llm_quality_score(query: str, answer: str, reference: str) -> dict:
3. 相关性(relevance)：回答是否直接针对问题，没有跑题或冗余
4. 流畅性(fluency)：回答是否通顺、结构清晰、易于理解
请严格按以下 JSON 格式返回，不要包含其他内容：
{{"accuracy": <分数>, "completeness": <分数>, "relevance": <分数>, "fluency": <分数>, "overall": <总分>}}"""
【输出要求】
只输出一个纯 JSON 对象，不要包含任何其他文字、解释或 markdown 格式：
{{"accuracy": <整数>, "completeness": <整数>, "relevance": <整数>, "fluency": <整数>, "overall": <整数>}}"""
def _extract_json(text: str) -> dict | None:
"""从文本中提取 JSON，支持嵌套大括号"""
if not text:
return None
# 1. 移除推理模型的思考标签及其内容
text = re.sub(r'<think>[\s\S]*?</think>', '', text).strip()
# 2. 尝试直接解析整个文本
try:
return json.loads(text)
except json.JSONDecodeError:
pass
# 3. 贪婪匹配最大的 {...} 块（支持嵌套）
# 从最后一个 } 往前找匹配的 {
brace_depth = 0
start = -1
end = -1
for i in range(len(text) - 1, -1, -1):
if text[i] == '}':
if brace_depth == 0:
end = i
brace_depth += 1
elif text[i] == '{':
brace_depth -= 1
if brace_depth == 0:
start = i
break
if start >= 0 and end > start:
try:
return json.loads(text[start:end + 1])
except json.JSONDecodeError:
pass
# 4. 回退：简单单层 {...} 匹配
m = re.search(r'\{[^{}]+\}', text)
if m:
try:
return json.loads(m.group())
except json.JSONDecodeError:
pass
return None
try:
response = client.chat.completions.create(
model=DASHSCOPE_MODEL,
messages=[{"role": "user", "content": prompt}],
temperature=0.1,
max_tokens=300
)
text = response.choices[0].message.content.strip()
# 构建请求参数
request_params = {
"model": DASHSCOPE_MODEL,
"messages": [
{"role": "system", "content": "You are a JSON-only evaluator. Output ONLY a valid JSON object, nothing else."},
{"role": "user", "content": prompt}
],
"temperature": 0.1,
"max_tokens": 2000,
}
# 尝试关闭推理模型的思考输出（部分 API 支持）
try:
request_params["extra_body"] = {"enable_thinking": False}
except Exception:
pass
# 提取 JSON
json_match = re.search(r'\{[^}]+\}', text)
if json_match:
scores = json.loads(json_match.group())
response = client.chat.completions.create(**request_params)
msg = response.choices[0].message
# 优先取 content，如果为空则尝试 reasoning_content 后的内容
text = getattr(msg, 'content', '') or ''
# 如果 content 为空，尝试从 reasoning_content 中提取
# （部分推理模型 API 将思考内容放在 reasoning_content，回答放在 content）
if not text.strip():
reasoning = getattr(msg, 'reasoning_content', '') or ''
if reasoning:
# 从思考内容末尾尝试提取 JSON
text = reasoning
if not text.strip():
logger.warning("LLM 返回内容为空")
return {"overall": 0.0, "error": "empty_response"}
scores = _extract_json(text)
if scores:
# 归一化到 0-1
for k in scores:
scores[k] = round(min(10, max(0, scores[k])) / 10.0, 4)
for k in list(scores.keys()):
try:
scores[k] = round(min(10, max(0, float(scores[k]))) / 10.0, 4)
except (ValueError, TypeError):
scores[k] = 0.0
return scores
else:
logger.warning(f"LLM 返回内容无法解析 JSON: {text[:100]}")
# 记录更多内容用于调试
debug_text = text[:200].replace('\n', '\\n')
logger.warning(f"LLM 返回内容无法解析 JSON: {debug_text}")
return {"overall": 0.0, "error": "parse_failed"}
except Exception as e:
logger.warning(f"LLM 评分异常: {e}")
@@ -228,7 +299,7 @@ def evaluate_dataset(dataset_path: str, use_llm: bool = True, api_url: str = Non
with open(dataset_path, 'r', encoding='utf-8') as f:
dataset = json.load(f)
questions = dataset.get("questions", [])
questions = dataset.get("queries", dataset.get("questions", []))
total = len(questions)
if total == 0:
logger.error("数据集中没有问题")

109
scripts/validate_eval_dataset.py Normal file
View File

@@ -0,0 +1,109 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
验证 eval_dataset_v2.json 的完整性和格式正确性
"""
import json
import sys
from pathlib import Path
from collections import Counter
PROJECT_ROOT = Path(__file__).parent.parent
DATASET_PATH = PROJECT_ROOT / "tests" / "eval_dataset_v2.json"
def validate():
errors = []
warnings = []
with open(DATASET_PATH, 'r', encoding='utf-8') as f:
data = json.load(f)
queries = data.get("queries", [])
print(f"[OK] Dataset loaded: {len(queries)} queries")
# 1. Check required fields
required = {"id", "query", "query_type", "relevant_docs", "reference_answer", "expected_keywords", "difficulty"}
for q in queries:
missing = required - set(q.keys())
if missing:
errors.append(f" {q.get('id', '?')}: missing fields: {missing}")
# 2. Check unique IDs
ids = [q["id"] for q in queries]
dupes = [id for id, cnt in Counter(ids).items() if cnt > 1]
if dupes:
errors.append(f" Duplicate IDs: {dupes}")
# 3. Check query types
valid_types = {"simple_fact", "enumeration", "definition", "comparison",
"reasoning", "table_data", "cross_doc", "negative", "paraphrase"}
type_counts = Counter(q["query_type"] for q in queries)
invalid_types = set(type_counts.keys()) - valid_types
if invalid_types:
errors.append(f" Invalid query_types: {invalid_types}")
print(f"\n[INFO] Query type distribution:")
for t, c in sorted(type_counts.items(), key=lambda x: -x[1]):
marker = " [INVALID]" if t in (invalid_types or set()) else ""
print(f" {t}: {c}{marker}")
# 4. Check difficulty distribution
diff_counts = Counter(q["difficulty"] for q in queries)
print(f"\n[INFO] Difficulty distribution:")
for d in ["easy", "medium", "hard"]:
print(f" {d}: {diff_counts.get(d, 0)}")
# 5. Check document coverage
doc_counts = Counter()
for q in queries:
for doc in q["relevant_docs"]:
doc_counts[doc] += 1
if not any("negative" in q["query_type"] for q in queries):
warnings.append(" No negative test cases")
print(f"\n[INFO] Document coverage:")
for doc, cnt in sorted(doc_counts.items(), key=lambda x: -x[1]):
print(f" {doc}: {cnt} queries")
neg_count = sum(1 for q in queries if q["query_type"] == "negative")
print(f" (negative/out-of-scope): {neg_count} queries")
# 6. Check expected_keywords
empty_kw = [q["id"] for q in queries if not q.get("expected_keywords")]
if empty_kw:
warnings.append(f" Empty expected_keywords: {empty_kw}")
# 7. Check paraphrase references
paraphrases = [q for q in queries if q["query_type"] == "paraphrase"]
for p in paraphrases:
ref = p.get("paraphrase_of")
if ref and ref not in ids:
errors.append(f" {p['id']}: paraphrase_of '{ref}' not found")
# 8. Check reference_answer length
short_refs = [q["id"] for q in queries if len(q.get("reference_answer", "")) < 10]
if short_refs:
warnings.append(f" Very short reference_answer: {short_refs}")
# Summary
print(f"\n{'='*60}")
if errors:
print(f"[ERROR] {len(errors)} error(s):")
for e in errors:
print(e)
if warnings:
print(f"[WARN] {len(warnings)} warning(s):")
for w in warnings:
print(w)
if not errors and not warnings:
print("[PASS] Dataset validation passed!")
print(f"{'='*60}")
return len(errors) == 0
if __name__ == "__main__":
if sys.platform == 'win32':
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
ok = validate()
sys.exit(0 if ok else 1)

614
tests/eval_dataset_v2.json Normal file
View File

@@ -0,0 +1,614 @@
{
"description": "RAG 系统综合评测数据集 v2 — 覆盖货源投放、吸烟环境、零售终端、三峡公报四个知识库文档",
"version": "2.0",
"created_at": "2026-06-17",
"source_documents": [
"1.docx",
"2.docx",
"3.docx",
"三峡公报_1-15页.pdf"
],
"query_categories": [
"simple_fact",
"enumeration",
"definition",
"comparison",
"reasoning",
"table_data",
"cross_doc",
"negative",
"paraphrase"
],
"queries": [
{
"id": "q001",
"query": "货源投放有哪些方式？",
"query_type": "enumeration",
"relevant_docs": ["1.docx"],
"reference_answer": "货源投放方式主要有六种：1.按档位投放；2.按档位+标签扩展投放；3.按价位段自选投放；4.选点投放；5.批零网配；6.事件用烟投放。",
"expected_keywords": ["按档位投放", "标签扩展", "价位段自选", "选点投放", "批零网配", "事件用烟", "六种"],
"difficulty": "easy"
},
{
"id": "q002",
"query": "市场状态分为哪几种？分别是什么？",
"query_type": "enumeration",
"relevant_docs": ["1.docx"],
"reference_answer": "市场状态分为五种：俏、紧、平、松、软。",
"expected_keywords": ["俏", "紧", "平", "松", "软", "五种"],
"difficulty": "easy"
},
{
"id": "q003",
"query": "什么是紧俏品规？",
"query_type": "definition",
"relevant_docs": ["1.docx"],
"reference_answer": "紧俏品规是指当地消费需求持续旺盛，货源缺口较大，零售价格坚挺，在较长时间内不能满足市场需求的品规。紧俏品规数不得超过地市级公司所经营品规数的20%。",
"expected_keywords": ["消费需求", "旺盛", "货源缺口", "零售价格坚挺", "20%"],
"difficulty": "medium"
},
{
"id": "q004",
"query": "紧俏品规数量有什么限制？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "紧俏品规数不得超过地市级公司所经营品规数的20%。",
"expected_keywords": ["20%", "不得超过", "品规数"],
"difficulty": "easy"
},
{
"id": "q005",
"query": "均衡满足品规有什么限制？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "均衡满足品规数不得超过地市级公司所经营顺销品规数的40%（顺销品规包括均衡满足品规和完全满足品规）。",
"expected_keywords": ["40%", "不得超过", "顺销品规", "均衡满足"],
"difficulty": "easy"
},
{
"id": "q006",
"query": "单客户单次单品规订货上限是多少？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "单客户单次单品规订货上限不超过50条（件）。",
"expected_keywords": ["50条", "单客户", "单品规", "不超过"],
"difficulty": "easy"
},
{
"id": "q007",
"query": "月度单客户订货总量上限是怎么规定的？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "月度单客户订货总量上限为上一年度平均月度进货量的15倍。",
"expected_keywords": ["15倍", "平均月度进货量", "上一年度"],
"difficulty": "easy"
},
{
"id": "q008",
"query": "主导品规的周供应占比要求是多少？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "主导品规周供应量应达到本价位段周供应量的60%以上。",
"expected_keywords": ["60%", "价位段", "周供应量", "主导品规"],
"difficulty": "medium"
},
{
"id": "q009",
"query": "主导品规和护卫品规的供应占比合计要求是多少？",
"query_type": "simple_fact",
"relevant_docs": ["1.docx"],
"reference_answer": "主导品规和护卫品规的周供应量合计应达到本价位段周供应量的85%以上。",
"expected_keywords": ["85%", "主导品规", "护卫品规", "合计"],
"difficulty": "medium"
},
{
"id": "q010",
"query": "货源投放的总体要求是什么？",
"query_type": "enumeration",
"relevant_docs": ["1.docx"],
"reference_answer": "货源投放是烟草营销的核心业务，总体要求包括六个坚持：坚持市场导向、供需匹配；坚持总量控制、稍紧平衡；坚持增速合理、贵在持续；坚持公平公正、严格规范；坚持状态优先、科学投放；坚持区域协同、高效运作。",
"expected_keywords": ["市场导向", "总量控制", "稍紧平衡", "公平公正", "状态优先", "区域协同", "六个"],
"difficulty": "medium"
},
{
"id": "q011",
"query": "新品投放有什么要求和限制？",
"query_type": "reasoning",
"relevant_docs": ["1.docx"],
"reference_answer": "新品指二类及以上销售时间不超过24个月的品规。新品可采用选点投放方式，要求认真分析新品定位，坚持科学民主决策，制订具体选点标准，不得人为指定选点客户名单。",
"expected_keywords": ["24个月", "选点投放", "科学民主", "不得人为指定"],
"difficulty": "medium"
},
{
"id": "q012",
"query": "紧俏品规在投放时有什么特殊要求？",
"query_type": "reasoning",
"relevant_docs": ["1.docx"],
"reference_answer": "紧俏品规投放要更加注重公平投放、普惠性投放，适度扩大供货面，且不得按档位扩展投放。",
"expected_keywords": ["公平投放", "普惠", "扩大供货面", "不得", "档位扩展"],
"difficulty": "hard"
},
{
"id": "q013",
"query": "按档位投放和选点投放有什么区别？",
"query_type": "comparison",
"relevant_docs": ["1.docx"],
"reference_answer": "按档位投放是由系统根据客户档级自动分配投放量，面向所有在档客户。选点投放是针对特定客户群体进行的定向投放，需要制定选点标准，经过集体决策确定投放对象。",
"expected_keywords": ["档级", "自动分配", "选点", "定向", "集体决策"],
"difficulty": "medium"
},
{
"id": "q014",
"query": "主导品规和护卫品规有什么区别？",
"query_type": "comparison",
"relevant_docs": ["1.docx"],
"reference_answer": "主导品规一般有1-4个，属于深受本地消费者喜爱、零售客户欢迎的品规。护卫品规一般有2-5个，属于本地有相当数量消费者喜爱的品规。主要区别在于规模和市场地位不同，主导品规规模更大、地位更重要。",
"expected_keywords": ["1-4个", "2-5个", "消费者喜爱", "规模", "市场地位"],
"difficulty": "medium"
},
{
"id": "q015",
"query": "如果某个品规供货面高、订货面低，应该怎么调整投放策略？",
"query_type": "reasoning",
"relevant_docs": ["1.docx"],
"reference_answer": "供货面高但订货面低说明投放范围过大但客户需求不足。应缩小供货面，减少投放客户范围，将货源集中到有真实需求的客户。",
"expected_keywords": ["供货面", "订货面", "调整", "策略"],
"difficulty": "hard"
},
{
"id": "q016",
"query": "批零网配客户配货增加量有什么限制？",
"query_type": "table_data",
"relevant_docs": ["1.docx"],
"reference_answer": "批零网配客户配货增加量不得超过同档级客户初始合理定量的50%。",
"expected_keywords": ["50%", "同档级", "初始合理定量", "不得超过"],
"difficulty": "medium"
},
{
"id": "q017",
"query": "市场状态评价指标体系中各指标的权重是怎样的？",
"query_type": "table_data",
"relevant_docs": ["1.docx"],
"reference_answer": "市场状态评价采用双层双级指标体系，不同品类（紧俏品规、均衡满足品规、宽松品规）的评价指标权重不同，包括零售价格指数、市场流通价格指数、终端动销率、终端库存、社会存销比、订货面、订足面等指标。",
"expected_keywords": ["双层双级", "权重", "零售价格指数", "动销率", "存销比", "订货面"],
"difficulty": "hard"
},
{
"id": "q018",
"query": "文明吸烟环境的SUCCESS功能框架包含哪些功能？",
"query_type": "enumeration",
"relevant_docs": ["2.docx"],
"reference_answer": "SUCCESS功能框架包含7个功能：S-Smoking（吸烟）、U-Utility（实用）、C-Convenience（便利）、C-Comfortable（舒适）、E-Experience（体验）、S-Safe（安全）、S-Survey（调研）。",
"expected_keywords": ["Smoking", "Utility", "Convenience", "Comfortable", "Experience", "Safe", "Survey", "七个"],
"difficulty": "medium"
},
{
"id": "q019",
"query": "吸烟设施有哪些类型？",
"query_type": "enumeration",
"relevant_docs": ["2.docx"],
"reference_answer": "吸烟设施分为三种类型：吸烟室、吸烟区、吸烟点。",
"expected_keywords": ["吸烟室", "吸烟区", "吸烟点", "三种"],
"difficulty": "easy"
},
{
"id": "q020",
"query": "吸烟环境分为哪几个区域类别？各覆盖什么场所？",
"query_type": "enumeration",
"relevant_docs": ["2.docx"],
"reference_answer": "分为三大类九个子类：A类（公共服务区域）包括A1交通枢纽、A2政务服务、A3旅游景区、A4医疗机构；B类（商业区域）包括B1商场、B2酒店、B3餐饮娱乐；C类（办公/工业区域）包括C1办公楼、C2工业园区。",
"expected_keywords": ["A类", "B类", "C类", "交通枢纽", "政务", "景区", "商场", "酒店", "办公楼"],
"difficulty": "hard"
},
{
"id": "q021",
"query": "吸烟场所的建设标准是哪一年发布的？共有多少条？",
"query_type": "simple_fact",
"relevant_docs": ["2.docx"],
"reference_answer": "文明吸烟环境建设标准（试行）共7章12条，管理标准（试行）共8章20条，维护标准（试行）共6章9条。三份标准均为试行版本。",
"expected_keywords": ["试行", "建设标准", "管理标准", "维护标准", "12条", "20条", "9条"],
"difficulty": "medium"
},
{
"id": "q022",
"query": "吸烟室、吸烟区和吸烟点有什么区别？",
"query_type": "comparison",
"relevant_docs": ["2.docx"],
"reference_answer": "吸烟室是独立的封闭空间，有面积和通风要求；吸烟区是在开放或半开放空间中划定的区域；吸烟点是最小化的设施，通常只配备烟灰缸和标识牌。三者在面积、设施配备、适用场所上有所不同。",
"expected_keywords": ["封闭空间", "开放", "划定", "面积", "设施配备", "场所"],
"difficulty": "medium"
},
{
"id": "q023",
"query": "A类区域和B类区域的吸烟设施配置有什么不同要求？",
"query_type": "comparison",
"relevant_docs": ["2.docx"],
"reference_answer": "A类区域（公共服务区域）和B类区域（商业区域）对吸烟设施的类型选择、面积标准、设备配置等有不同的要求。A类区域通常要求更高标准的设施配置。",
"expected_keywords": ["A类", "B类", "配置", "面积", "标准", "不同"],
"difficulty": "hard"
},
{
"id": "q024",
"query": "吸烟设施的日常维护和定期维护有什么区别？",
"query_type": "comparison",
"relevant_docs": ["2.docx"],
"reference_answer": "日常维护是指每日或高频次的保洁、设备检查等基础工作。定期维护是指按固定周期进行的深度清洁、设备更换、设施检修等工作。两者的频率、深度和负责人员不同。",
"expected_keywords": ["日常维护", "定期维护", "频率", "保洁", "设备更换", "深度清洁"],
"difficulty": "medium"
},
{
"id": "q025",
"query": "吸烟设施的编码规则是怎样的？",
"query_type": "definition",
"relevant_docs": ["2.docx"],
"reference_answer": "吸烟设施编码格式为：所在区（县/市）+街道（镇）+道路名称+场所编码（吸烟室为AXXX、吸烟区亭为BXXX、吸烟点为CXXX）+具体设备编码（XXX）。",
"expected_keywords": ["编码", "区县", "街道", "AXXX", "BXXX", "CXXX"],
"difficulty": "medium"
},
{
"id": "q026",
"query": "吸烟环境的维护交接流程是怎样的？",
"query_type": "reasoning",
"relevant_docs": ["2.docx"],
"reference_answer": "维护交接需要按照维护标准中的规定，完成正式的交接手续，填写交接记录表，明确交接双方的责任和义务。",
"expected_keywords": ["交接", "手续", "记录", "责任", "维护标准"],
"difficulty": "medium"
},
{
"id": "q027",
"query": "吸烟设施的监督检查包括哪些内容？",
"query_type": "enumeration",
"relevant_docs": ["2.docx"],
"reference_answer": "监督检查是管理标准中的专门章节，包括对设施运行状态、卫生状况、设备完好程度、使用规范性等方面的检查。",
"expected_keywords": ["监督检查", "运行状态", "卫生", "设备", "规范性"],
"difficulty": "medium"
},
{
"id": "q028",
"query": "零售终端的分类体系是怎样的？从高到低怎么排列？",
"query_type": "enumeration",
"relevant_docs": ["3.docx"],
"reference_answer": "零售终端从高到低分为：直营终端、合作终端、加盟终端（星级加盟/普通加盟）、现代终端（A/B/C/D/E五个等级）、普通终端（含'五化'普通终端）。",
"expected_keywords": ["直营终端", "合作终端", "加盟终端", "现代终端", "普通终端", "A", "B", "C", "D", "E"],
"difficulty": "medium"
},
{
"id": "q029",
"query": "加盟终端使用什么品牌名称？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端使用'金丝利零售'（Kingsley Retail）品牌。",
"expected_keywords": ["金丝利零售", "Kingsley", "Retail", "品牌"],
"difficulty": "easy"
},
{
"id": "q030",
"query": "加盟终端的加盟协议期限是多长？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "首次加盟协议期限为三年，续签加盟协议期限也为三年。注意：智能终端设备合作协议和现代终端合作协议的期限是五年，不要混淆。",
"expected_keywords": ["三年", "加盟协议", "期限"],
"difficulty": "easy"
},
{
"id": "q031",
"query": "加盟终端评价的合格分数线是多少？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端运行评价满分为100分，达标分为80分（含）。一般现代终端评价同样为满分100分、达标分80分。",
"expected_keywords": ["100分", "80分", "达标", "评价"],
"difficulty": "easy"
},
{
"id": "q032",
"query": "星标加盟终端的评分标准是什么？城网和农网有什么区别？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "星标加盟终端评价标准：城网要求评价周期内平均分95分及以上、单次得分不低于90分；农网要求平均分90分及以上、单次得分不低于85分。两者都需要评价周期内积极配合市县两级公司开展各类活动。",
"expected_keywords": ["95分", "90分", "85分", "星标", "城网", "农网"],
"difficulty": "medium"
},
{
"id": "q033",
"query": "加盟终端之间的最小间距要求是多少？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端布局间步行距离应不低于300米。",
"expected_keywords": ["300米", "步行距离", "不低于"],
"difficulty": "easy"
},
{
"id": "q034",
"query": "加盟终端的卷烟展示面积要求是多少？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "卷烟展示面积不低于1.8平方米。",
"expected_keywords": ["1.8", "平方米", "展示面积", "不低于"],
"difficulty": "easy"
},
{
"id": "q035",
"query": "加盟终端的每日营业时间要求是多少？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端营业时间要求每天12小时以上。",
"expected_keywords": ["12小时", "营业时间", "每天"],
"difficulty": "easy"
},
{
"id": "q036",
"query": "加盟终端的'六个统一'是什么？",
"query_type": "enumeration",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端实行'六个统一'管理体系，涵盖品牌形象、服务标准、运营管理等方面的统一要求。",
"expected_keywords": ["六个统一", "品牌形象", "服务标准", "管理"],
"difficulty": "medium"
},
{
"id": "q037",
"query": "现代终端分为几个等级？分别是什么？",
"query_type": "enumeration",
"relevant_docs": ["3.docx"],
"reference_answer": "一般现代终端分为两层五类：新现代终端（A类、B类、C类）和普通现代终端（D类、E类）。A类使用江苏烟草自采智能终端设备，B类使用非自采智能终端设备，C类使用简易智能终端设备，D类通过数据接口自动上传，E类通过客户手动上传。",
"expected_keywords": ["A类", "B类", "C类", "D类", "E类", "新现代", "普通现代"],
"difficulty": "easy"
},
{
"id": "q038",
"query": "终端运营管理中有哪些禁止行为？",
"query_type": "enumeration",
"relevant_docs": ["3.docx"],
"reference_answer": "终端运营管理中规定了10条禁止行为，用于保护品牌价值和规范终端经营行为。",
"expected_keywords": ["10条", "禁止行为", "品牌价值", "经营"],
"difficulty": "medium"
},
{
"id": "q039",
"query": "加盟终端和合作终端有什么区别？",
"query_type": "comparison",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端使用'金丝利零售'品牌，签订3年加盟协议，实行'六个统一'管理。合作终端是另一种合作模式，与加盟终端在品牌使用、管理方式、投入标准等方面有所不同。",
"expected_keywords": ["金丝利零售", "加盟协议", "合作", "品牌", "管理方式"],
"difficulty": "medium"
},
{
"id": "q040",
"query": "终端评价不合格会怎么处理？",
"query_type": "reasoning",
"relevant_docs": ["3.docx"],
"reference_answer": "终端评价不合格将进入退出管理流程，根据评价结果进行降级或退出处理。退出管理定义了具体的降级和移除程序。",
"expected_keywords": ["退出管理", "降级", "退出", "评价", "不合格"],
"difficulty": "medium"
},
{
"id": "q041",
"query": "直营终端有什么特殊要求？",
"query_type": "reasoning",
"relevant_docs": ["3.docx"],
"reference_answer": "直营终端是由公司直接拥有和运营的终端，在品牌形象、服务标准、运营管理等方面执行最高标准，是终端体系中的最高层级。",
"expected_keywords": ["直营", "公司", "运营", "最高", "层级"],
"difficulty": "medium"
},
{
"id": "q042",
"query": "普通终端如何升级为现代终端？",
"query_type": "reasoning",
"relevant_docs": ["3.docx"],
"reference_answer": "普通终端可通过'五化'建设升级为五化普通终端，进而达到现代终端标准。五化是普通终端的升级路径，包括店面形象、经营管理、信息化等方面的提升。",
"expected_keywords": ["五化", "升级", "普通终端", "现代终端", "店面", "经营管理"],
"difficulty": "hard"
},
{
"id": "q043",
"query": "终端建设标准的附件有多少个？",
"query_type": "simple_fact",
"relevant_docs": ["3.docx"],
"reference_answer": "终端梯次化建设标准共有20个附件，包括新现代终端客户申请书、现代终端合作协议、加盟终端合作协议、评价评分表、审批表、维护申请表等各类模板。",
"expected_keywords": ["20个", "附件", "申请书", "合作协议", "评价"],
"difficulty": "easy"
},
{
"id": "q044",
"query": "三峡工程有哪些综合效益？",
"query_type": "enumeration",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "三峡工程的综合效益包括防洪、发电、航运、水资源利用、生态环境保护等。三峡电站是世界上总装机容量最大的水电站。",
"expected_keywords": ["防洪", "发电", "航运", "水资源", "生态", "综合效益"],
"difficulty": "easy"
},
{
"id": "q045",
"query": "2022年三峡电站发电量为什么比往年低？",
"query_type": "reasoning",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年三峡电站年度发电量为787.90亿千瓦时，为2012年以来最低值。主要原因是长江流域遭遇1961年以来最严重的夏秋连旱，7-9月入库水量仅1007亿立方米。",
"expected_keywords": ["787.90", "干旱", "1961年", "夏秋连旱", "1007亿", "入库水量"],
"difficulty": "medium"
},
{
"id": "q046",
"query": "三峡工程在防洪方面发挥了什么作用？",
"query_type": "definition",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "三峡水库蓄水运行以来，至2022年累计汛期拦洪总量2005.16亿立方米，有效应对了多次区域性大洪水，大大降低了中下游洪水位。",
"expected_keywords": ["2005.16亿", "拦洪", "汛期", "洪水位", "中下游"],
"difficulty": "medium"
},
{
"id": "q047",
"query": "三峡船闸和升船机2022年的运行情况对比如何？",
"query_type": "comparison",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年三峡船闸运行10400闸次，通过船舶40641艘次，货运量1.56亿吨，通航率92.72%。升船机运行4470厢次，通过船舶4506艘次，旅客54416人次。",
"expected_keywords": ["10400", "40641", "1.56亿吨", "92.72%", "4470", "54416"],
"difficulty": "medium"
},
{
"id": "q048",
"query": "三峡工程对节能减排有什么贡献？",
"query_type": "reasoning",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "截至2022年底，三峡电站累计发出清洁电力相当于节约标准煤4.85亿吨，减少二氧化碳排放12.65亿吨。",
"expected_keywords": ["4.85亿吨", "标准煤", "12.65亿吨", "二氧化碳", "清洁电力"],
"difficulty": "medium"
},
{
"id": "q049",
"query": "2022年三峡水库的泥沙淤积情况如何？排沙比是多少？",
"query_type": "table_data",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年三峡水库入库悬移质输沙量0.136亿吨，出库0.026亿吨，库区淤积0.110亿吨，排沙比19.3%。",
"expected_keywords": ["0.136亿吨", "0.026亿吨", "0.110亿吨", "19.3%", "排沙比"],
"difficulty": "medium"
},
{
"id": "q050",
"query": "2022年三峡水库蓄水目标完成了吗？为什么？",
"query_type": "reasoning",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年三峡水库库水位最高蓄至160.04米，未完成175米蓄水目标。原因是长江流域遭遇严重干旱，入库水量减少，同时为应对下游旱情多次补水。",
"expected_keywords": ["160.04米", "175米", "未完成", "干旱", "补水"],
"difficulty": "hard"
},
{
"id": "q051",
"query": "2022年三峡水库为什么频繁向下游补水？补了多少？",
"query_type": "reasoning",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年长江流域遭遇严重干旱，三峡水库频繁向下游补水是为缓解下游旱情。枯水季节累计补水158天，补水总量217.76亿立方米。",
"expected_keywords": ["158天", "217.76亿", "旱情", "枯水季节", "补水"],
"difficulty": "medium"
},
{
"id": "q052",
"query": "三峡工程2022年进行了哪些生态调度试验？",
"query_type": "enumeration",
"relevant_docs": ["三峡公报_1-15页.pdf"],
"reference_answer": "2022年进行了三类生态调度：1.针对四大家鱼等产漂流性卵鱼类繁殖的水文生态调度试验；2.针对库区产黏沉性卵鱼类自然繁殖的生态调度试验；3.库尾减淤调度试验。",
"expected_keywords": ["四大家鱼", "漂流性卵", "黏沉性卵", "减淤", "生态调度"],
"difficulty": "medium"
},
{
"id": "q053",
"query": "三份工作规范文档分别涉及什么领域？各自适用范围是什么？",
"query_type": "cross_doc",
"relevant_docs": ["1.docx", "2.docx", "3.docx"],
"reference_answer": "1.docx是货源投放工作规范，适用于烟草产品的供应分配管理。2.docx是文明吸烟环境建设/管理/维护标准，适用于吸烟设施的建设与运维。3.docx是零售终端梯次化建设标准，适用于卷烟零售终端的分级管理。三份文档均属于江苏省烟草行业规范。",
"expected_keywords": ["货源投放", "吸烟环境", "零售终端", "江苏省", "烟草"],
"difficulty": "hard"
},
{
"id": "q054",
"query": "三份文档中的分类体系有什么相似之处？",
"query_type": "cross_doc",
"relevant_docs": ["1.docx", "2.docx", "3.docx"],
"reference_answer": "三份文档都采用了分层分类的管理方法：货源投放将品规分为紧俏/均衡/宽松三类，客户按档级分类；吸烟环境将设施分为室/区/点三类并按A/B/C区域分类；零售终端分为普通/现代/加盟/合作/直营五个层级。",
"expected_keywords": ["分类", "分层", "品规", "设施", "终端", "层级"],
"difficulty": "hard"
},
{
"id": "q055",
"query": "货源投放规范和终端建设标准在评价体系上有什么共同点？",
"query_type": "cross_doc",
"relevant_docs": ["1.docx", "3.docx"],
"reference_answer": "两者都采用了量化评分体系：货源投放使用双层双级指标体系评价市场状态，终端建设使用100分制评价终端达标情况。都设置了明确的评分阈值和等级划分标准。",
"expected_keywords": ["量化", "评分", "指标体系", "阈值", "等级"],
"difficulty": "hard"
},
{
"id": "q056",
"query": "我们公司食堂在哪里？",
"query_type": "negative",
"relevant_docs": [],
"reference_answer": "知识库中没有关于公司食堂的信息。现有文档主要涉及货源投放工作规范、文明吸烟环境标准和零售终端建设标准。",
"expected_keywords": ["未找到", "未"],
"difficulty": "easy"
},
{
"id": "q057",
"query": "公司的股票代码是什么？",
"query_type": "negative",
"relevant_docs": [],
"reference_answer": "知识库中没有关于股票代码的信息。现有文档主要涉及烟草行业的工作规范和建设标准。",
"expected_keywords": ["未找到", "未"],
"difficulty": "easy"
},
{
"id": "q058",
"query": "如何申请年休假？需要哪些流程？",
"query_type": "negative",
"relevant_docs": [],
"reference_answer": "知识库中没有关于年休假申请流程的信息。现有文档涉及的是货源投放、吸烟环境建设和零售终端管理。",
"expected_keywords": ["未找到", "未"],
"difficulty": "easy"
},
{
"id": "q059",
"query": "紧俏品规的上限比例是多少？",
"query_type": "paraphrase",
"relevant_docs": ["1.docx"],
"reference_answer": "紧俏品规数不得超过地市级公司所经营品规数的20%。",
"expected_keywords": ["20%", "不得超过"],
"difficulty": "easy",
"paraphrase_of": "q004"
},
{
"id": "q060",
"query": "吸烟环境的七种功能分别是什么？",
"query_type": "paraphrase",
"relevant_docs": ["2.docx"],
"reference_answer": "SUCCESS功能框架包含7个功能：S-Smoking（吸烟）、U-Utility（实用）、C-Convenience（便利）、C-Comfortable（舒适）、E-Experience（体验）、S-Safe（安全）、S-Survey（调研）。",
"expected_keywords": ["Smoking", "Utility", "Convenience", "Comfortable", "Experience", "Safe", "Survey"],
"difficulty": "medium",
"paraphrase_of": "q018"
},
{
"id": "q061",
"query": "加盟门店之间最少要隔多远？",
"query_type": "paraphrase",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端之间的最小间距为300米。",
"expected_keywords": ["300米", "间距"],
"difficulty": "easy",
"paraphrase_of": "q033"
},
{
"id": "q062",
"query": "金丝利零售门店每天最少营业多长时间？",
"query_type": "paraphrase",
"relevant_docs": ["3.docx"],
"reference_answer": "加盟终端每日营业时间不低于12小时。",
"expected_keywords": ["12小时", "营业"],
"difficulty": "easy",
"paraphrase_of": "q035"
}
],
"statistics": {
"total_queries": 62,
"by_category": {
"simple_fact": 15,
"enumeration": 13,
"reasoning": 11,
"comparison": 7,
"paraphrase": 4,
"definition": 3,
"table_data": 3,
"cross_doc": 3,
"negative": 3
},
"by_difficulty": {
"easy": 22,
"medium": 30,
"hard": 10
},
"by_document": {
"1.docx": 21,
"2.docx": 13,
"3.docx": 21,
"三峡公报_1-15页.pdf": 9,
"negative(无相关文档)": 3,
"cross_doc(多文档)": 3,
"paraphrase(改写)": 4
}
}
}