diff --git a/api/chat_routes.py b/api/chat_routes.py index 9fb587f..3eb2275 100644 --- a/api/chat_routes.py +++ b/api/chat_routes.py @@ -493,6 +493,198 @@ def _section_similarity(section_a: str, section_b: str) -> float: return overlap / union if union > 0 else 0.0 +def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str, + min_score: float) -> List[Dict]: + """ + 词法匹配救援：当切片文本精确包含查询的核心关键词但 CrossEncoder 评分很低时， + 将其分数提升至保底值，并同时救援同 source 下 chunk_index 相邻的切片。 + + 适用场景： + 1. 某个 section 只有一个正确切片（无法触发聚类救援） + 2. 枚举类问题的 header 切片被词法匹配救援后，其后续子条目也应被一并保留 + + Args: + contexts: 全部上下文切片 + retrieval_query: 检索查询 + min_score: 最低分数阈值 + + Returns: + 修改后的 contexts + """ + if not contexts or not retrieval_query: + return contexts + + import re + from config import CLUSTER_RESCUE_FLOOR + + # 清理查询：去除 markdown 格式和标点 + clean_query = re.sub(r'\*+|#+|`', '', retrieval_query) + clean_query = re.sub(r'[？?！!。，,、；;：:"""\'\s]+', ' ', clean_query).strip() + + if len(clean_query) < 2: + return contexts + + # 提取查询中的有意义 bigram（连续两字组） + query_bigrams = set() + for i in range(len(clean_query) - 1): + w = clean_query[i:i+2].strip() + if len(w) == 2: + query_bigrams.add(w) + + if not query_bigrams: + return contexts + + # Phase 1: 词法匹配救援——找到高分匹配的切片 + rescued_seeds = [] # [(source, chunk_index)] + for ctx in contexts: + if ctx.get('score', 0) >= min_score: + continue + + doc = ctx.get('doc', '') or '' + meta = ctx.get('meta', {}) + section = meta.get('section', '') or meta.get('section_path', '') + combined = doc + ' ' + section + + matched = sum(1 for w in query_bigrams if w in combined) + match_ratio = matched / len(query_bigrams) + + if match_ratio > 0.35: + ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR) + source = meta.get('source', '') + chunk_index = meta.get('chunk_index') + if source and chunk_index is not None: + try: + rescued_seeds.append((source, int(chunk_index))) + except (ValueError, TypeError): + pass + + # Phase 2: 邻居救援——对每个被词法匹配救援的种子， + # 同时救援同 source 下 chunk_index 后续相邻的切片（枚举子条目） + if rescued_seeds: + for source, seed_idx in rescued_seeds: + for ctx in contexts: + if ctx.get('score', 0) >= min_score: + continue + meta = ctx.get('meta', {}) + if meta.get('source') != source: + continue + try: + n_idx = int(meta.get('chunk_index', -1)) + except (ValueError, TypeError): + continue + # 救援种子后续 8 个相邻切片（覆盖大多数枚举/条款模式） + if seed_idx < n_idx <= seed_idx + 8: + ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR) + + return contexts + + +def _normalize_section_for_rescue(section_path: str, levels: int = 2) -> str: + """归一化 section_path：取前 N 级路径用于分组""" + if not section_path: + return '' + parts = [p.strip() for p in section_path.split('>')] + return ' > '.join(parts[:levels]) + + +def _rescue_section_cluster(contexts: List[Dict], retrieval_query: str, + min_score: float) -> List[Dict]: + """ + 路由层章节聚类救援：当某个 section 有多个候选切片但全部低于 min_score 时， + 给该 section 的切片分配保底分数，使其通过后续的 min_score 过滤。 + + 作为引擎层 _section_cluster_boost 的二次安全网，防止极端情况下所有正确切片被过滤。 + + Args: + contexts: engine 返回的全部上下文切片（每个含 doc, meta, score） + retrieval_query: 改写后的检索查询 + min_score: 当前的最低分数阈值 + + Returns: + 修改后的 contexts（部分切片 score 被提升至保底分数） + """ + if not contexts: + return contexts + + from config import ( + CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, CLUSTER_RESCUE_FLOOR, + CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION, + CLUSTER_SECTION_PREFIX_LEVELS, + ) + + # 1. 按 (source, normalized_section) 分组 + from collections import defaultdict + section_groups = defaultdict(list) # key → [index_in_contexts] + + for i, ctx in enumerate(contexts): + meta = ctx.get('meta', {}) + source = meta.get('source', '') + section_path = meta.get('section', '') or meta.get('section_path', '') + norm_section = _normalize_section_for_rescue(section_path, CLUSTER_SECTION_PREFIX_LEVELS) + if not source or not norm_section: + continue + key = (source, norm_section) + section_groups[key].append(i) + + # 2. 检测"全灭 section"并计算聚类强度 + rescue_candidates = [] # (strength, key, member_indices) + + for key, indices in section_groups.items(): + if len(indices) < CLUSTER_MIN_MEMBERS: + continue + + # 检查是否所有成员都低于 min_score（"全灭"） + scores = [contexts[i].get('score', 0) for i in indices] + if any(s >= min_score for s in scores): + continue # 已有成员通过阈值，无需救援 + + # 计算聚类强度 + chunk_types = set(contexts[i].get('meta', {}).get('chunk_type', 'text') for i in indices) + type_diversity = len(chunk_types) + if type_diversity < CLUSTER_MIN_TYPES: + continue # 类型不够多样，可能是噪音 + + # 查询与 section_path 的字符重叠率 + source, norm_section = key + query_chars = set(retrieval_query) + section_chars = set(norm_section) + overlap = len(query_chars & section_chars) / max(len(query_chars), 1) + + # 聚类强度 = 成员数 × 类型多样性 × (1 + 查询匹配度) + strength = len(indices) * type_diversity * (1.0 + overlap) + rescue_candidates.append((strength, key, indices)) + + # 3. 按强度降序，救援 top-N section + rescue_candidates.sort(key=lambda x: x[0], reverse=True) + + rescued_sections = [] + total_rescued = 0 + + for strength, key, indices in rescue_candidates[:CLUSTER_MAX_SECTIONS]: + # 对组内切片分配保底分数（仅低于 min_score 的） + rescue_count = 0 + for idx in indices: + if rescue_count >= CLUSTER_MAX_RESCUE_PER_SECTION: + break + ctx = contexts[idx] + if ctx.get('score', 0) < min_score: + ctx['score'] = CLUSTER_RESCUE_FLOOR + rescue_count += 1 + total_rescued += 1 + + if rescue_count > 0: + source, norm_section = key + rescued_sections.append({ + 'source': source, + 'section': norm_section, + 'members': len(indices), + 'rescued': rescue_count, + 'strength': round(strength, 2) + }) + + return contexts + + def _rescue_table_chunks(contexts: List[Dict], context_text: str, retrieval_query: str, max_rescue_chars: int = 3000) -> str: """ @@ -1731,7 +1923,10 @@ def rag(): DIRECT_CONTEXT_MAX_CHARS, RERANK_CONTEXT_MIN_SCORE, CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT, - CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD + CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD, + SECTION_CLUSTER_RESCUE_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, + CLUSTER_RESCUE_FLOOR, CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION, + CLUSTER_SECTION_PREFIX_LEVELS ) data = request.json or {} @@ -1973,6 +2168,11 @@ def rag(): debug_info = search_result.get('_debug', {}) yield f"data: {json.dumps({'type': 'retrieval_debug', 'data': {'steps': debug_info.get('steps', []), 'collections_searched': collections, 'total_candidates': len(search_result.get('ids', [[]])[0])}}, ensure_ascii=False)}\n\n" + # 提取章节聚类提升事件（引擎层）单独发送 + for step in debug_info.get('steps', []): + if step.get('name') == 'section_cluster_boost': + yield f"data: {json.dumps({'type': 'section_cluster_boost', 'data': step}, ensure_ascii=False)}\n\n" + # 提取上下文 contexts = [] sources = [] @@ -2241,6 +2441,32 @@ def rag(): # 4. 构建 prompt（Phase 6：LLM 图片感知） # Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争 + + # 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时， + # 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景） + contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE) + + # 章节聚类救援（路由层安全网）：在 min_score 过滤前， + # 检测"全灭 section"并分配保底分数 + _rescue_debug = None + if SECTION_CLUSTER_RESCUE_ENABLED: + _before_scores = {id(ctx): ctx.get('score', 0) for ctx in contexts} + contexts = _rescue_section_cluster(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE) + # 统计救援结果 + _rescued = [ctx for ctx in contexts if id(ctx) in _before_scores + and ctx.get('score', 0) > _before_scores[id(ctx)]] + if _rescued and IS_DEV: + _rescue_sections = set() + for ctx in _rescued: + meta = ctx.get('meta', {}) + sp = meta.get('section', '') or meta.get('section_path', '') + _rescue_sections.add(_normalize_section_for_rescue(sp, CLUSTER_SECTION_PREFIX_LEVELS)) + _rescue_debug = { + 'rescued_count': len(_rescued), + 'rescued_sections': list(_rescue_sections) + } + yield f"data: {json.dumps({'type': 'section_cluster_rescue', 'data': _rescue_debug}, ensure_ascii=False)}\n\n" + text_contexts = _order_text_contexts_for_prompt(contexts, retrieval_query, MAX_CONTEXT_CHUNKS, min_score=RERANK_CONTEXT_MIN_SCORE) # Phase 2：按字符预算构建上下文 diff --git a/core/engine.py b/core/engine.py index 9805ecd..4e3b878 100644 --- a/core/engine.py +++ b/core/engine.py @@ -77,6 +77,10 @@ try: CONTEXT_EXPANSION_MAX_CHUNKS, ENUM_QUERY_DISABLE_TOPK_SHRINK, ENUM_QUERY_MMR_LAMBDA, # Phase 3 扩展精细化 EXPANSION_SCORE_THRESHOLD, MAX_EXPANDED_NEIGHBORS, + # 章节聚类救援 + SECTION_CLUSTER_BOOST_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, + CLUSTER_SEED_FLOOR, CLUSTER_MAX_BOOST_PER_SECTION, CLUSTER_MAX_SECTIONS, + CLUSTER_SECTION_PREFIX_LEVELS, # 上下文与生成 LLM_TEMPERATURE, LLM_MAX_TOKENS, RECALL_MULTIPLIER, # FAQ 与黑名单 @@ -102,10 +106,18 @@ except ImportError: MMR_TOP_K = 30 CONTEXT_EXPANSION_ENABLED = True CONTEXT_EXPANSION_BEFORE = 1 - CONTEXT_EXPANSION_AFTER = 5 + CONTEXT_EXPANSION_AFTER = 8 CONTEXT_EXPANSION_MAX_CHUNKS = 24 EXPANSION_SCORE_THRESHOLD = 0.3 - MAX_EXPANDED_NEIGHBORS = 4 + MAX_EXPANDED_NEIGHBORS = 8 + # 章节聚类救援默认值 + SECTION_CLUSTER_BOOST_ENABLED = True + CLUSTER_MIN_MEMBERS = 3 + CLUSTER_MIN_TYPES = 2 + CLUSTER_SEED_FLOOR = 0.35 + CLUSTER_MAX_BOOST_PER_SECTION = 8 + CLUSTER_MAX_SECTIONS = 3 + CLUSTER_SECTION_PREFIX_LEVELS = 1 ENUM_QUERY_DISABLE_TOPK_SHRINK = True ENUM_QUERY_MMR_LAMBDA = 0.85 DYNAMIC_RRF_ENABLED = True @@ -732,11 +744,19 @@ class RAGEngine: # 时间衰减（Time Decay） fused_results = self._apply_time_decay(fused_results) + # 提前附加 _debug，使聚类提升能写入调试步骤 + fused_results['_debug'] = _debug + + # ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ========== + if SECTION_CLUSTER_BOOST_ENABLED: + fused_results = self._section_cluster_boost(fused_results, query) + # ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）========== # Phase 3：仅对高分种子扩展邻居 before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0 fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k, - min_score=EXPANSION_SCORE_THRESHOLD) + min_score=EXPANSION_SCORE_THRESHOLD, + query=query) after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0 _debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp}) @@ -1160,14 +1180,139 @@ class RAGEngine: return None return self.collection + @staticmethod + def _normalize_section_path(section_path: str, levels: int = None) -> str: + """归一化 section_path：取前 N 级路径，容忍 MinerU 标题检测误差。 + + 例如: "第三章 吸烟场所的功能设置 > 第三条 文明吸烟..." → "第三章 吸烟场所的功能设置" + """ + if not section_path: + return '' + if levels is None: + levels = CLUSTER_SECTION_PREFIX_LEVELS + parts = [p.strip() for p in section_path.split('>')] + return ' > '.join(parts[:levels]) + + def _section_cluster_boost(self, results: dict, query: str = '') -> dict: + """章节聚类提升：当同一 section 下多个切片（text+table）同时出现在候选集中， + 即使单个切片 CrossEncoder 分数很低，也将整组提升到种子阈值。 + + 核心洞察：单个低分切片不可信，但同一 section 多个切片同时出现是强信号。 + 提升后的切片可以作为 _expand_contiguous_chunks 的种子，触发邻居扩展。 + + Args: + results: rerank 后的检索结果 + query: 用户查询（用于后续扩展） + + Returns: + 修改后的 results（distances 被调整，meta 中标记 _cluster_boosted） + """ + if not results.get('ids') or not results['ids'][0]: + return results + + ids = results['ids'][0] + metas = results.get('metadatas', [[]])[0] + distances = results.get('distances', [[]])[0] if results.get('distances') else None + + if not distances: + return results + + # 1. 按 (source, normalized_section) 分组 + from collections import defaultdict + section_groups = defaultdict(list) # key → [(index, meta, dist)] + + for i, (meta, dist) in enumerate(zip(metas, distances)): + source = meta.get('source', '') + section_path = meta.get('section', '') or meta.get('section_path', '') + norm_section = self._normalize_section_path(section_path) + if not source or not norm_section: + continue + key = (source, norm_section) + section_groups[key].append((i, meta, dist)) + + # 2. 检测聚类信号并提升 + boost_target_dist = 1.0 - CLUSTER_SEED_FLOOR # score=0.35 → dist=0.65 + boosted_sections = [] + total_boosted = 0 + + # 按组成员数降序排列，优先处理最大聚类 + sorted_groups = sorted(section_groups.items(), key=lambda x: len(x[1]), reverse=True) + + for (source, norm_section), members in sorted_groups: + if len(boosted_sections) >= CLUSTER_MAX_SECTIONS: + break + + # 聚类信号检测：成员数 >= 阈值 且 类型多样性 >= 阈值 + chunk_types = set(m[1].get('chunk_type', 'text') for m in members) + if len(members) < CLUSTER_MIN_MEMBERS or len(chunk_types) < CLUSTER_MIN_TYPES: + continue + + # 提升组内切片分数（仅提升低于阈值的） + boost_count = 0 + for idx, meta, dist in members: + if boost_count >= CLUSTER_MAX_BOOST_PER_SECTION: + break + # 只提升分数低于种子阈值的切片（高分切片不需要） + if dist > boost_target_dist: + distances[idx] = boost_target_dist + meta['_cluster_boosted'] = True + boost_count += 1 + total_boosted += 1 + + if boost_count > 0: + boosted_sections.append({ + 'source': source, + 'section': norm_section, + 'members': len(members), + 'types': list(chunk_types), + 'boosted': boost_count + }) + + # 3. 写 debug 信息 + if boosted_sections: + debug_info = results.get('_debug', {}) + if 'steps' not in debug_info: + debug_info['steps'] = [] + debug_info['steps'].append({ + 'name': 'section_cluster_boost', + 'sections': boosted_sections, + 'total_boosted': total_boosted + }) + results['_debug'] = debug_info + logger.info(f"[章节聚类提升] 提升 {total_boosted} 个切片，" + f"涉及 {len(boosted_sections)} 个 section: " + f"{[s['section'][:30] for s in boosted_sections]}") + + return results + + @staticmethod + def _chunk_lexical_score(chunk_text: str, query: str) -> float: + """计算切片文本与查询的词法重叠度（bigram 命中率），用于辅助种子资格判定。""" + if not chunk_text or not query: + return 0.0 + import re + clean_q = re.sub(r'[？?！!。，,、；;：:"""\'\s*#`]+', ' ', query).strip() + if len(clean_q) < 2: + return 0.0 + bigrams = set() + for i in range(len(clean_q) - 1): + w = clean_q[i:i+2].strip() + if len(w) == 2: + bigrams.add(w) + if not bigrams: + return 0.0 + matched = sum(1 for w in bigrams if w in chunk_text) + return matched / len(bigrams) + def _expand_contiguous_chunks(self, results: dict, top_k: int = None, - min_score: float = 0.0) -> dict: + min_score: float = 0.0, query: str = '') -> dict: """Add same-source same-section neighbor text chunks around strong hits. Args: results: 检索结果 top_k: 最大切片数 min_score: Phase 3 最低分数阈值，仅对 Rerank 分数高于此值的种子扩展 + query: 查询文本，用于词法匹配辅助种子资格判定 """ if not CONTEXT_EXPANSION_ENABLED: return results @@ -1197,7 +1342,7 @@ class RAGEngine: seeds = [ (doc_id, doc, meta, dist) for doc_id, doc, meta, dist in items[:base_limit] - if meta.get('chunk_type', 'text') == 'text' + if (meta.get('chunk_type', 'text') == 'text' or meta.get('_cluster_boosted')) and meta.get('source') and self._to_int(meta.get('chunk_index')) is not None ] @@ -1208,8 +1353,12 @@ class RAGEngine: break # Phase 3：跳过分数低于阈值的种子（仅当 min_score > 0 时生效） + # 词法匹配豁免：CrossEncoder 低分但关键词重叠度高时仍允许作为种子 if min_score > 0 and seed_dist < min_score: - continue + if query and self._chunk_lexical_score(_seed_doc, query) > 0.3: + pass # 词法匹配度高，允许作为种子 + else: + continue source = seed_meta.get('source') section = seed_meta.get('section', '') or seed_meta.get('section_path', '') @@ -1627,11 +1776,19 @@ class RAGEngine: # 时间衰减 fused_results = self._apply_time_decay(fused_results) + # 提前附加 _debug，使聚类提升能写入调试步骤 + fused_results['_debug'] = _debug + + # ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ========== + if SECTION_CLUSTER_BOOST_ENABLED: + fused_results = self._section_cluster_boost(fused_results, query) + # ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）========== # Phase 3：仅对高分种子扩展邻居 before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0 fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k, - min_score=EXPANSION_SCORE_THRESHOLD) + min_score=EXPANSION_SCORE_THRESHOLD, + query=query) after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0 if _debug is not None: _debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp}) diff --git a/core/mmr.py b/core/mmr.py index 7e8c29e..4565fd1 100644 --- a/core/mmr.py +++ b/core/mmr.py @@ -108,22 +108,44 @@ def mmr_rerank( return selected +def _tokenize_words(text: str) -> set: + """ + 使用 jieba 分词并过滤噪声，返回有意义的词集合。 + + 过滤规则： + - 去除单字符词（如 "的", "了", "在"）—— 这些是停用词，对区分文档无意义 + - 去除纯数字 / 纯标点 + - 保留 2 字及以上的实词 + """ + import jieba + words = set() + for w in jieba.cut(text): + w = w.strip() + if len(w) >= 2 and not w.isdigit(): + words.add(w) + return words + + def mmr_filter_by_content( candidates: List[Dict], top_k: int = 30, - similarity_threshold: float = 0.9 + similarity_threshold: float = 0.85 ) -> List[Dict]: """ - 基于内容相似度的去重（简化版，不需要 embedding） + 基于 jieba 词级 Jaccard 相似度的去重（不需要 embedding） + + 与旧版字符级 set(text) 的区别： + - 旧版：set("安全生产管理制度") → {'安','全','生','产',...}，中文文档间字符集合高度重叠 + - 新版：jieba 分词 → {"安全生产", "管理制度", ...}，词级集合区分度高 适用于： - - 没有 embedding 的情况 - - 快速去重场景 + - MMR_USE_EMBEDDING=False 时的快速去重 + - 避免 CPU 编码 100+ 文档的 50 秒开销 Args: candidates: 候选文档列表 top_k: 返回数量 - similarity_threshold: 相似度阈值，超过则视为重复 + similarity_threshold: 相似度阈值，超过则视为重复（默认 0.85） Returns: 去重后的候选文档列表 @@ -134,35 +156,42 @@ def mmr_filter_by_content( if len(candidates) <= top_k: return candidates - selected = [] - remaining = candidates.copy() + # 预分词：对所有候选文档一次性分词，避免重复调用 jieba.cut + word_sets = [] + for c in candidates: + content = c.get('content', c.get('document', ''))[:500] + word_sets.append(_tokenize_words(content)) - while len(selected) < top_k and remaining: - current = remaining.pop(0) + selected_indices = [] + + for i in range(len(candidates)): + if len(selected_indices) >= top_k: + break + + current_words = word_sets[i] + if not current_words: + # 空内容直接保留 + selected_indices.append(i) + continue - # 检查是否与已选内容重复 is_duplicate = False - current_content = current.get('content', current.get('document', ''))[:200] + for j in selected_indices: + selected_words = word_sets[j] + if not selected_words: + continue - for s in selected: - s_content = s.get('content', s.get('document', ''))[:200] + intersection = len(current_words & selected_words) + union = len(current_words | selected_words) + similarity = intersection / union if union > 0 else 0 - # 简单的 Jaccard 相似度 - words1 = set(current_content) - words2 = set(s_content) - if words1 and words2: - intersection = len(words1 & words2) - union = len(words1 | words2) - similarity = intersection / union if union > 0 else 0 - - if similarity > similarity_threshold: - is_duplicate = True - break + if similarity > similarity_threshold: + is_duplicate = True + break if not is_duplicate: - selected.append(current) + selected_indices.append(i) - return selected + return [candidates[i] for i in selected_indices] # ==================== 测试 ==================== diff --git a/knowledge/base.py b/knowledge/base.py index 814207b..5210c27 100644 --- a/knowledge/base.py +++ b/knowledge/base.py @@ -382,18 +382,34 @@ class BM25Index: def add_documents(self, ids: List[str], documents: List[str], metadatas: List[dict]) -> None: """ - 添加文档到索引（会覆盖原有索引） + 添加文档到索引（追加模式，自动去重） + + 如果 ID 已存在则更新对应文档，否则追加新文档。 + 添加后自动重建 BM25 索引。 Args: ids: 文档 ID 列表 documents: 文档内容列表 metadatas: 文档元数据列表 """ - self.ids = ids - self.documents = documents - self.metadatas = metadatas - if documents: - tokenized = [self.tokenize(doc) for doc in documents] + # 建立已有 ID -> 索引位置 的映射，用于去重 + existing_map = {doc_id: idx for idx, doc_id in enumerate(self.ids)} + + for i, doc_id in enumerate(ids): + if doc_id in existing_map: + # 更新已有文档 + pos = existing_map[doc_id] + self.documents[pos] = documents[i] + self.metadatas[pos] = metadatas[i] + else: + # 追加新文档 + existing_map[doc_id] = len(self.ids) + self.ids.append(doc_id) + self.documents.append(documents[i]) + self.metadatas.append(metadatas[i]) + + if self.documents: + tokenized = [self.tokenize(doc) for doc in self.documents] self.bm25 = BM25Okapi(tokenized) def search(self, query: str, top_k: int = 10) -> Tuple[List[str], List[str], List[dict], List[float]]: diff --git a/parsers/mineru_parser.py b/parsers/mineru_parser.py index 94cc517..99451a0 100644 --- a/parsers/mineru_parser.py +++ b/parsers/mineru_parser.py @@ -137,8 +137,8 @@ def parse_with_mineru_online( file_path: str, api_token: str = None, api_url: str = None, - model_version: str = "vlm", - timeout: int = 300 + model_version: str = None, + timeout: int = None ) -> Dict[str, Any]: """ 使用 MinerU 在线 API 解析文档 @@ -157,8 +157,8 @@ def parse_with_mineru_online( file_path: 文档文件路径 api_token: API Token（默认从 config 读取） api_url: API 地址 - model_version: 模型版本 (vlm / pipeline / MinerU-HTML) - timeout: 请求超时（秒） + model_version: 模型版本 (vlm / pipeline / MinerU-HTML)，默认从 config 读取 + timeout: 轮询超时（秒），默认从 config 读取 Returns: 解析结果（与 parse_with_mineru 格式相同） @@ -167,10 +167,12 @@ def parse_with_mineru_online( import time import zipfile import io - from config import MINERU_API_TOKEN, MINERU_API_URL + from config import MINERU_API_TOKEN, MINERU_API_URL, MINERU_MODEL_VERSION, MINERU_ONLINE_TIMEOUT token = api_token or MINERU_API_TOKEN url = api_url or MINERU_API_URL + model_version = model_version or MINERU_MODEL_VERSION + timeout = timeout or MINERU_ONLINE_TIMEOUT if not token: raise RuntimeError("MinerU 在线 API Token 未配置，请在 config.py 中设置 MINERU_API_TOKEN") @@ -241,9 +243,15 @@ def parse_with_mineru_online( result_resp.raise_for_status() result = result_resp.json() + # 检查 API 层面的错误码，快速失败而非静默等到超时 + api_code = result.get("code") + if api_code and api_code != 0: + api_msg = result.get("msg", "未知错误") + raise RuntimeError(f"MinerU API 错误 (code={api_code}): {api_msg}") + extract_results = result.get("data", {}).get("extract_result", []) if not extract_results: - logger.debug(f"等待解析结果... ({waited}s)") + logger.debug(f"等待解析结果... ({waited}s/{max_wait}s)") continue # 取第一个文件的结果 @@ -276,11 +284,16 @@ def parse_with_mineru_online( if progress: extracted = progress.get("extracted_pages", 0) total = progress.get("total_pages", 0) - logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s)") + logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s/{max_wait}s, model={model_version})") else: - logger.debug(f"状态: {state}, 等待中... ({waited}s)") + logger.debug(f"状态: {state}, 等待中... ({waited}s/{max_wait}s)") - raise RuntimeError("MinerU 在线解析超时") + raise RuntimeError( + f"MinerU 在线解析超时 ({max_wait}s)" + f"，当前 model_version={model_version}" + f"，可尝试: 1) 设置 MINERU_MODEL_VERSION=pipeline 加速" + f" 2) 增大 MINERU_ONLINE_TIMEOUT" + ) except Exception as e: raise RuntimeError(f"MinerU 在线 API 调用失败: {e}") @@ -737,7 +750,7 @@ def parse_with_mineru( lang: str = "ch", enable_table: bool = True, enable_formula: bool = True, - backend: str = "pipeline", + backend: str = None, start_page: int = 0, end_page: int = 99999 ) -> Dict[str, Any]: @@ -770,6 +783,14 @@ def parse_with_mineru( if not file_path.exists(): raise FileNotFoundError(f"文件不存在: {file_path}") + # 从 config 读取默认 backend + if backend is None: + try: + from config import MINERU_LOCAL_BACKEND + backend = MINERU_LOCAL_BACKEND + except ImportError: + backend = 'pipeline' + # 检查文件大小 file_size = file_path.stat().st_size if file_size > MAX_PDF_SIZE: @@ -814,7 +835,6 @@ def parse_with_mineru( cmd = [ str(mineru_exe), - "--", "-p", str(file_path), "-o", str(output_dir), "-m", "auto", @@ -859,7 +879,7 @@ def parse_with_mineru( logger.error(f"MinerU 解析失败: {e}") raise finally: - 清理临时目录 + # 清理临时目录 if cleanup_output and os.path.exists(output_dir): shutil.rmtree(output_dir, ignore_errors=True) @@ -1688,7 +1708,7 @@ def parse_with_mineru_persistent( lang: str = "ch", enable_table: bool = True, enable_formula: bool = True, - backend: str = "pipeline", + backend: str = None, start_page: int = 0, end_page: int = 99999, cleanup_after_image_move: bool = True @@ -1727,6 +1747,14 @@ def parse_with_mineru_persistent( if not file_path.exists(): raise FileNotFoundError(f"文件不存在: {file_path}") + # 从 config 读取默认 backend + if backend is None: + try: + from config import MINERU_LOCAL_BACKEND + backend = MINERU_LOCAL_BACKEND + except ImportError: + backend = 'pipeline' + # 检查文件大小 file_size = file_path.stat().st_size if file_size > MAX_PDF_SIZE: diff --git a/scripts/eval_e2e.py b/scripts/eval_e2e.py index b2ebaf9..3538f5c 100644 --- a/scripts/eval_e2e.py +++ b/scripts/eval_e2e.py @@ -54,7 +54,7 @@ def setup_file_logging(log_path: str): # ───────────── 配置 ───────────── RAG_API_URL = "http://127.0.0.1:5001/rag" -DEFAULT_DATASET = "data/eval/eval_dataset.json" +DEFAULT_DATASET = "tests/eval_dataset_v2.json" RESULTS_DIR = "data/eval_results" REQUEST_TIMEOUT = 120 # 秒 REQUEST_INTERVAL = 1.5 # 请求间隔（秒），避免过载 @@ -79,7 +79,8 @@ def call_rag_sse(question: str, collections: list = None, api_url: str = None) - headers = { "Content-Type": "application/json", - "Accept": "text/event-stream" + "Accept": "text/event-stream", + "Authorization": "Bearer mock-token-admin" } # chat_history 在生产模式下是必填字段 payload = {"message": question, "chat_history": []} @@ -185,28 +186,98 @@ def llm_quality_score(query: str, answer: str, reference: str) -> dict: 3. 相关性(relevance)：回答是否直接针对问题，没有跑题或冗余 4. 流畅性(fluency)：回答是否通顺、结构清晰、易于理解 -请严格按以下 JSON 格式返回，不要包含其他内容： -{{"accuracy": <分数>, "completeness": <分数>, "relevance": <分数>, "fluency": <分数>, "overall": <总分>}}""" +【输出要求】 +只输出一个纯 JSON 对象，不要包含任何其他文字、解释或 markdown 格式： +{{"accuracy": <整数>, "completeness": <整数>, "relevance": <整数>, "fluency": <整数>, "overall": <整数>}}""" + + def _extract_json(text: str) -> dict | None: + """从文本中提取 JSON，支持嵌套大括号""" + if not text: + return None + # 1. 移除推理模型的思考标签及其内容 + text = re.sub(r'[\s\S]*?', '', text).strip() + # 2. 尝试直接解析整个文本 + try: + return json.loads(text) + except json.JSONDecodeError: + pass + # 3. 贪婪匹配最大的 {...} 块（支持嵌套） + # 从最后一个 } 往前找匹配的 { + brace_depth = 0 + start = -1 + end = -1 + for i in range(len(text) - 1, -1, -1): + if text[i] == '}': + if brace_depth == 0: + end = i + brace_depth += 1 + elif text[i] == '{': + brace_depth -= 1 + if brace_depth == 0: + start = i + break + if start >= 0 and end > start: + try: + return json.loads(text[start:end + 1]) + except json.JSONDecodeError: + pass + # 4. 回退：简单单层 {...} 匹配 + m = re.search(r'\{[^{}]+\}', text) + if m: + try: + return json.loads(m.group()) + except json.JSONDecodeError: + pass + return None try: - response = client.chat.completions.create( - model=DASHSCOPE_MODEL, - messages=[{"role": "user", "content": prompt}], - temperature=0.1, - max_tokens=300 - ) - text = response.choices[0].message.content.strip() + # 构建请求参数 + request_params = { + "model": DASHSCOPE_MODEL, + "messages": [ + {"role": "system", "content": "You are a JSON-only evaluator. Output ONLY a valid JSON object, nothing else."}, + {"role": "user", "content": prompt} + ], + "temperature": 0.1, + "max_tokens": 2000, + } + # 尝试关闭推理模型的思考输出（部分 API 支持） + try: + request_params["extra_body"] = {"enable_thinking": False} + except Exception: + pass - # 提取 JSON - json_match = re.search(r'\{[^}]+\}', text) - if json_match: - scores = json.loads(json_match.group()) + response = client.chat.completions.create(**request_params) + msg = response.choices[0].message + + # 优先取 content，如果为空则尝试 reasoning_content 后的内容 + text = getattr(msg, 'content', '') or '' + + # 如果 content 为空，尝试从 reasoning_content 中提取 + # （部分推理模型 API 将思考内容放在 reasoning_content，回答放在 content） + if not text.strip(): + reasoning = getattr(msg, 'reasoning_content', '') or '' + if reasoning: + # 从思考内容末尾尝试提取 JSON + text = reasoning + + if not text.strip(): + logger.warning("LLM 返回内容为空") + return {"overall": 0.0, "error": "empty_response"} + + scores = _extract_json(text) + if scores: # 归一化到 0-1 - for k in scores: - scores[k] = round(min(10, max(0, scores[k])) / 10.0, 4) + for k in list(scores.keys()): + try: + scores[k] = round(min(10, max(0, float(scores[k]))) / 10.0, 4) + except (ValueError, TypeError): + scores[k] = 0.0 return scores else: - logger.warning(f"LLM 返回内容无法解析 JSON: {text[:100]}") + # 记录更多内容用于调试 + debug_text = text[:200].replace('\n', '\\n') + logger.warning(f"LLM 返回内容无法解析 JSON: {debug_text}") return {"overall": 0.0, "error": "parse_failed"} except Exception as e: logger.warning(f"LLM 评分异常: {e}") @@ -228,7 +299,7 @@ def evaluate_dataset(dataset_path: str, use_llm: bool = True, api_url: str = Non with open(dataset_path, 'r', encoding='utf-8') as f: dataset = json.load(f) - questions = dataset.get("questions", []) + questions = dataset.get("queries", dataset.get("questions", [])) total = len(questions) if total == 0: logger.error("数据集中没有问题") diff --git a/scripts/validate_eval_dataset.py b/scripts/validate_eval_dataset.py new file mode 100644 index 0000000..54b689e --- /dev/null +++ b/scripts/validate_eval_dataset.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +验证 eval_dataset_v2.json 的完整性和格式正确性 +""" +import json +import sys +from pathlib import Path +from collections import Counter + +PROJECT_ROOT = Path(__file__).parent.parent +DATASET_PATH = PROJECT_ROOT / "tests" / "eval_dataset_v2.json" + + +def validate(): + errors = [] + warnings = [] + + with open(DATASET_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + + queries = data.get("queries", []) + print(f"[OK] Dataset loaded: {len(queries)} queries") + + # 1. Check required fields + required = {"id", "query", "query_type", "relevant_docs", "reference_answer", "expected_keywords", "difficulty"} + for q in queries: + missing = required - set(q.keys()) + if missing: + errors.append(f" {q.get('id', '?')}: missing fields: {missing}") + + # 2. Check unique IDs + ids = [q["id"] for q in queries] + dupes = [id for id, cnt in Counter(ids).items() if cnt > 1] + if dupes: + errors.append(f" Duplicate IDs: {dupes}") + + # 3. Check query types + valid_types = {"simple_fact", "enumeration", "definition", "comparison", + "reasoning", "table_data", "cross_doc", "negative", "paraphrase"} + type_counts = Counter(q["query_type"] for q in queries) + invalid_types = set(type_counts.keys()) - valid_types + if invalid_types: + errors.append(f" Invalid query_types: {invalid_types}") + + print(f"\n[INFO] Query type distribution:") + for t, c in sorted(type_counts.items(), key=lambda x: -x[1]): + marker = " [INVALID]" if t in (invalid_types or set()) else "" + print(f" {t}: {c}{marker}") + + # 4. Check difficulty distribution + diff_counts = Counter(q["difficulty"] for q in queries) + print(f"\n[INFO] Difficulty distribution:") + for d in ["easy", "medium", "hard"]: + print(f" {d}: {diff_counts.get(d, 0)}") + + # 5. Check document coverage + doc_counts = Counter() + for q in queries: + for doc in q["relevant_docs"]: + doc_counts[doc] += 1 + if not any("negative" in q["query_type"] for q in queries): + warnings.append(" No negative test cases") + + print(f"\n[INFO] Document coverage:") + for doc, cnt in sorted(doc_counts.items(), key=lambda x: -x[1]): + print(f" {doc}: {cnt} queries") + neg_count = sum(1 for q in queries if q["query_type"] == "negative") + print(f" (negative/out-of-scope): {neg_count} queries") + + # 6. Check expected_keywords + empty_kw = [q["id"] for q in queries if not q.get("expected_keywords")] + if empty_kw: + warnings.append(f" Empty expected_keywords: {empty_kw}") + + # 7. Check paraphrase references + paraphrases = [q for q in queries if q["query_type"] == "paraphrase"] + for p in paraphrases: + ref = p.get("paraphrase_of") + if ref and ref not in ids: + errors.append(f" {p['id']}: paraphrase_of '{ref}' not found") + + # 8. Check reference_answer length + short_refs = [q["id"] for q in queries if len(q.get("reference_answer", "")) < 10] + if short_refs: + warnings.append(f" Very short reference_answer: {short_refs}") + + # Summary + print(f"\n{'='*60}") + if errors: + print(f"[ERROR] {len(errors)} error(s):") + for e in errors: + print(e) + if warnings: + print(f"[WARN] {len(warnings)} warning(s):") + for w in warnings: + print(w) + if not errors and not warnings: + print("[PASS] Dataset validation passed!") + print(f"{'='*60}") + + return len(errors) == 0 + + +if __name__ == "__main__": + if sys.platform == 'win32': + sys.stdout.reconfigure(encoding='utf-8', errors='replace') + ok = validate() + sys.exit(0 if ok else 1) diff --git a/tests/eval_dataset_v2.json b/tests/eval_dataset_v2.json new file mode 100644 index 0000000..e7d465e --- /dev/null +++ b/tests/eval_dataset_v2.json @@ -0,0 +1,614 @@ +{ + "description": "RAG 系统综合评测数据集 v2 — 覆盖货源投放、吸烟环境、零售终端、三峡公报四个知识库文档", + "version": "2.0", + "created_at": "2026-06-17", + "source_documents": [ + "1.docx", + "2.docx", + "3.docx", + "三峡公报_1-15页.pdf" + ], + "query_categories": [ + "simple_fact", + "enumeration", + "definition", + "comparison", + "reasoning", + "table_data", + "cross_doc", + "negative", + "paraphrase" + ], + "queries": [ + { + "id": "q001", + "query": "货源投放有哪些方式？", + "query_type": "enumeration", + "relevant_docs": ["1.docx"], + "reference_answer": "货源投放方式主要有六种：1.按档位投放；2.按档位+标签扩展投放；3.按价位段自选投放；4.选点投放；5.批零网配；6.事件用烟投放。", + "expected_keywords": ["按档位投放", "标签扩展", "价位段自选", "选点投放", "批零网配", "事件用烟", "六种"], + "difficulty": "easy" + }, + { + "id": "q002", + "query": "市场状态分为哪几种？分别是什么？", + "query_type": "enumeration", + "relevant_docs": ["1.docx"], + "reference_answer": "市场状态分为五种：俏、紧、平、松、软。", + "expected_keywords": ["俏", "紧", "平", "松", "软", "五种"], + "difficulty": "easy" + }, + { + "id": "q003", + "query": "什么是紧俏品规？", + "query_type": "definition", + "relevant_docs": ["1.docx"], + "reference_answer": "紧俏品规是指当地消费需求持续旺盛，货源缺口较大，零售价格坚挺，在较长时间内不能满足市场需求的品规。紧俏品规数不得超过地市级公司所经营品规数的20%。", + "expected_keywords": ["消费需求", "旺盛", "货源缺口", "零售价格坚挺", "20%"], + "difficulty": "medium" + }, + { + "id": "q004", + "query": "紧俏品规数量有什么限制？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "紧俏品规数不得超过地市级公司所经营品规数的20%。", + "expected_keywords": ["20%", "不得超过", "品规数"], + "difficulty": "easy" + }, + { + "id": "q005", + "query": "均衡满足品规有什么限制？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "均衡满足品规数不得超过地市级公司所经营顺销品规数的40%（顺销品规包括均衡满足品规和完全满足品规）。", + "expected_keywords": ["40%", "不得超过", "顺销品规", "均衡满足"], + "difficulty": "easy" + }, + { + "id": "q006", + "query": "单客户单次单品规订货上限是多少？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "单客户单次单品规订货上限不超过50条（件）。", + "expected_keywords": ["50条", "单客户", "单品规", "不超过"], + "difficulty": "easy" + }, + { + "id": "q007", + "query": "月度单客户订货总量上限是怎么规定的？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "月度单客户订货总量上限为上一年度平均月度进货量的15倍。", + "expected_keywords": ["15倍", "平均月度进货量", "上一年度"], + "difficulty": "easy" + }, + { + "id": "q008", + "query": "主导品规的周供应占比要求是多少？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "主导品规周供应量应达到本价位段周供应量的60%以上。", + "expected_keywords": ["60%", "价位段", "周供应量", "主导品规"], + "difficulty": "medium" + }, + { + "id": "q009", + "query": "主导品规和护卫品规的供应占比合计要求是多少？", + "query_type": "simple_fact", + "relevant_docs": ["1.docx"], + "reference_answer": "主导品规和护卫品规的周供应量合计应达到本价位段周供应量的85%以上。", + "expected_keywords": ["85%", "主导品规", "护卫品规", "合计"], + "difficulty": "medium" + }, + { + "id": "q010", + "query": "货源投放的总体要求是什么？", + "query_type": "enumeration", + "relevant_docs": ["1.docx"], + "reference_answer": "货源投放是烟草营销的核心业务，总体要求包括六个坚持：坚持市场导向、供需匹配；坚持总量控制、稍紧平衡；坚持增速合理、贵在持续；坚持公平公正、严格规范；坚持状态优先、科学投放；坚持区域协同、高效运作。", + "expected_keywords": ["市场导向", "总量控制", "稍紧平衡", "公平公正", "状态优先", "区域协同", "六个"], + "difficulty": "medium" + }, + { + "id": "q011", + "query": "新品投放有什么要求和限制？", + "query_type": "reasoning", + "relevant_docs": ["1.docx"], + "reference_answer": "新品指二类及以上销售时间不超过24个月的品规。新品可采用选点投放方式，要求认真分析新品定位，坚持科学民主决策，制订具体选点标准，不得人为指定选点客户名单。", + "expected_keywords": ["24个月", "选点投放", "科学民主", "不得人为指定"], + "difficulty": "medium" + }, + { + "id": "q012", + "query": "紧俏品规在投放时有什么特殊要求？", + "query_type": "reasoning", + "relevant_docs": ["1.docx"], + "reference_answer": "紧俏品规投放要更加注重公平投放、普惠性投放，适度扩大供货面，且不得按档位扩展投放。", + "expected_keywords": ["公平投放", "普惠", "扩大供货面", "不得", "档位扩展"], + "difficulty": "hard" + }, + { + "id": "q013", + "query": "按档位投放和选点投放有什么区别？", + "query_type": "comparison", + "relevant_docs": ["1.docx"], + "reference_answer": "按档位投放是由系统根据客户档级自动分配投放量，面向所有在档客户。选点投放是针对特定客户群体进行的定向投放，需要制定选点标准，经过集体决策确定投放对象。", + "expected_keywords": ["档级", "自动分配", "选点", "定向", "集体决策"], + "difficulty": "medium" + }, + { + "id": "q014", + "query": "主导品规和护卫品规有什么区别？", + "query_type": "comparison", + "relevant_docs": ["1.docx"], + "reference_answer": "主导品规一般有1-4个，属于深受本地消费者喜爱、零售客户欢迎的品规。护卫品规一般有2-5个，属于本地有相当数量消费者喜爱的品规。主要区别在于规模和市场地位不同，主导品规规模更大、地位更重要。", + "expected_keywords": ["1-4个", "2-5个", "消费者喜爱", "规模", "市场地位"], + "difficulty": "medium" + }, + { + "id": "q015", + "query": "如果某个品规供货面高、订货面低，应该怎么调整投放策略？", + "query_type": "reasoning", + "relevant_docs": ["1.docx"], + "reference_answer": "供货面高但订货面低说明投放范围过大但客户需求不足。应缩小供货面，减少投放客户范围，将货源集中到有真实需求的客户。", + "expected_keywords": ["供货面", "订货面", "调整", "策略"], + "difficulty": "hard" + }, + { + "id": "q016", + "query": "批零网配客户配货增加量有什么限制？", + "query_type": "table_data", + "relevant_docs": ["1.docx"], + "reference_answer": "批零网配客户配货增加量不得超过同档级客户初始合理定量的50%。", + "expected_keywords": ["50%", "同档级", "初始合理定量", "不得超过"], + "difficulty": "medium" + }, + { + "id": "q017", + "query": "市场状态评价指标体系中各指标的权重是怎样的？", + "query_type": "table_data", + "relevant_docs": ["1.docx"], + "reference_answer": "市场状态评价采用双层双级指标体系，不同品类（紧俏品规、均衡满足品规、宽松品规）的评价指标权重不同，包括零售价格指数、市场流通价格指数、终端动销率、终端库存、社会存销比、订货面、订足面等指标。", + "expected_keywords": ["双层双级", "权重", "零售价格指数", "动销率", "存销比", "订货面"], + "difficulty": "hard" + }, + { + "id": "q018", + "query": "文明吸烟环境的SUCCESS功能框架包含哪些功能？", + "query_type": "enumeration", + "relevant_docs": ["2.docx"], + "reference_answer": "SUCCESS功能框架包含7个功能：S-Smoking（吸烟）、U-Utility（实用）、C-Convenience（便利）、C-Comfortable（舒适）、E-Experience（体验）、S-Safe（安全）、S-Survey（调研）。", + "expected_keywords": ["Smoking", "Utility", "Convenience", "Comfortable", "Experience", "Safe", "Survey", "七个"], + "difficulty": "medium" + }, + { + "id": "q019", + "query": "吸烟设施有哪些类型？", + "query_type": "enumeration", + "relevant_docs": ["2.docx"], + "reference_answer": "吸烟设施分为三种类型：吸烟室、吸烟区、吸烟点。", + "expected_keywords": ["吸烟室", "吸烟区", "吸烟点", "三种"], + "difficulty": "easy" + }, + { + "id": "q020", + "query": "吸烟环境分为哪几个区域类别？各覆盖什么场所？", + "query_type": "enumeration", + "relevant_docs": ["2.docx"], + "reference_answer": "分为三大类九个子类：A类（公共服务区域）包括A1交通枢纽、A2政务服务、A3旅游景区、A4医疗机构；B类（商业区域）包括B1商场、B2酒店、B3餐饮娱乐；C类（办公/工业区域）包括C1办公楼、C2工业园区。", + "expected_keywords": ["A类", "B类", "C类", "交通枢纽", "政务", "景区", "商场", "酒店", "办公楼"], + "difficulty": "hard" + }, + { + "id": "q021", + "query": "吸烟场所的建设标准是哪一年发布的？共有多少条？", + "query_type": "simple_fact", + "relevant_docs": ["2.docx"], + "reference_answer": "文明吸烟环境建设标准（试行）共7章12条，管理标准（试行）共8章20条，维护标准（试行）共6章9条。三份标准均为试行版本。", + "expected_keywords": ["试行", "建设标准", "管理标准", "维护标准", "12条", "20条", "9条"], + "difficulty": "medium" + }, + { + "id": "q022", + "query": "吸烟室、吸烟区和吸烟点有什么区别？", + "query_type": "comparison", + "relevant_docs": ["2.docx"], + "reference_answer": "吸烟室是独立的封闭空间，有面积和通风要求；吸烟区是在开放或半开放空间中划定的区域；吸烟点是最小化的设施，通常只配备烟灰缸和标识牌。三者在面积、设施配备、适用场所上有所不同。", + "expected_keywords": ["封闭空间", "开放", "划定", "面积", "设施配备", "场所"], + "difficulty": "medium" + }, + { + "id": "q023", + "query": "A类区域和B类区域的吸烟设施配置有什么不同要求？", + "query_type": "comparison", + "relevant_docs": ["2.docx"], + "reference_answer": "A类区域（公共服务区域）和B类区域（商业区域）对吸烟设施的类型选择、面积标准、设备配置等有不同的要求。A类区域通常要求更高标准的设施配置。", + "expected_keywords": ["A类", "B类", "配置", "面积", "标准", "不同"], + "difficulty": "hard" + }, + { + "id": "q024", + "query": "吸烟设施的日常维护和定期维护有什么区别？", + "query_type": "comparison", + "relevant_docs": ["2.docx"], + "reference_answer": "日常维护是指每日或高频次的保洁、设备检查等基础工作。定期维护是指按固定周期进行的深度清洁、设备更换、设施检修等工作。两者的频率、深度和负责人员不同。", + "expected_keywords": ["日常维护", "定期维护", "频率", "保洁", "设备更换", "深度清洁"], + "difficulty": "medium" + }, + { + "id": "q025", + "query": "吸烟设施的编码规则是怎样的？", + "query_type": "definition", + "relevant_docs": ["2.docx"], + "reference_answer": "吸烟设施编码格式为：所在区（县/市）+街道（镇）+道路名称+场所编码（吸烟室为AXXX、吸烟区亭为BXXX、吸烟点为CXXX）+具体设备编码（XXX）。", + "expected_keywords": ["编码", "区县", "街道", "AXXX", "BXXX", "CXXX"], + "difficulty": "medium" + }, + { + "id": "q026", + "query": "吸烟环境的维护交接流程是怎样的？", + "query_type": "reasoning", + "relevant_docs": ["2.docx"], + "reference_answer": "维护交接需要按照维护标准中的规定，完成正式的交接手续，填写交接记录表，明确交接双方的责任和义务。", + "expected_keywords": ["交接", "手续", "记录", "责任", "维护标准"], + "difficulty": "medium" + }, + { + "id": "q027", + "query": "吸烟设施的监督检查包括哪些内容？", + "query_type": "enumeration", + "relevant_docs": ["2.docx"], + "reference_answer": "监督检查是管理标准中的专门章节，包括对设施运行状态、卫生状况、设备完好程度、使用规范性等方面的检查。", + "expected_keywords": ["监督检查", "运行状态", "卫生", "设备", "规范性"], + "difficulty": "medium" + }, + { + "id": "q028", + "query": "零售终端的分类体系是怎样的？从高到低怎么排列？", + "query_type": "enumeration", + "relevant_docs": ["3.docx"], + "reference_answer": "零售终端从高到低分为：直营终端、合作终端、加盟终端（星级加盟/普通加盟）、现代终端（A/B/C/D/E五个等级）、普通终端（含'五化'普通终端）。", + "expected_keywords": ["直营终端", "合作终端", "加盟终端", "现代终端", "普通终端", "A", "B", "C", "D", "E"], + "difficulty": "medium" + }, + { + "id": "q029", + "query": "加盟终端使用什么品牌名称？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端使用'金丝利零售'（Kingsley Retail）品牌。", + "expected_keywords": ["金丝利零售", "Kingsley", "Retail", "品牌"], + "difficulty": "easy" + }, + { + "id": "q030", + "query": "加盟终端的加盟协议期限是多长？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "首次加盟协议期限为三年，续签加盟协议期限也为三年。注意：智能终端设备合作协议和现代终端合作协议的期限是五年，不要混淆。", + "expected_keywords": ["三年", "加盟协议", "期限"], + "difficulty": "easy" + }, + { + "id": "q031", + "query": "加盟终端评价的合格分数线是多少？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端运行评价满分为100分，达标分为80分（含）。一般现代终端评价同样为满分100分、达标分80分。", + "expected_keywords": ["100分", "80分", "达标", "评价"], + "difficulty": "easy" + }, + { + "id": "q032", + "query": "星标加盟终端的评分标准是什么？城网和农网有什么区别？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "星标加盟终端评价标准：城网要求评价周期内平均分95分及以上、单次得分不低于90分；农网要求平均分90分及以上、单次得分不低于85分。两者都需要评价周期内积极配合市县两级公司开展各类活动。", + "expected_keywords": ["95分", "90分", "85分", "星标", "城网", "农网"], + "difficulty": "medium" + }, + { + "id": "q033", + "query": "加盟终端之间的最小间距要求是多少？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端布局间步行距离应不低于300米。", + "expected_keywords": ["300米", "步行距离", "不低于"], + "difficulty": "easy" + }, + { + "id": "q034", + "query": "加盟终端的卷烟展示面积要求是多少？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "卷烟展示面积不低于1.8平方米。", + "expected_keywords": ["1.8", "平方米", "展示面积", "不低于"], + "difficulty": "easy" + }, + { + "id": "q035", + "query": "加盟终端的每日营业时间要求是多少？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端营业时间要求每天12小时以上。", + "expected_keywords": ["12小时", "营业时间", "每天"], + "difficulty": "easy" + }, + { + "id": "q036", + "query": "加盟终端的'六个统一'是什么？", + "query_type": "enumeration", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端实行'六个统一'管理体系，涵盖品牌形象、服务标准、运营管理等方面的统一要求。", + "expected_keywords": ["六个统一", "品牌形象", "服务标准", "管理"], + "difficulty": "medium" + }, + { + "id": "q037", + "query": "现代终端分为几个等级？分别是什么？", + "query_type": "enumeration", + "relevant_docs": ["3.docx"], + "reference_answer": "一般现代终端分为两层五类：新现代终端（A类、B类、C类）和普通现代终端（D类、E类）。A类使用江苏烟草自采智能终端设备，B类使用非自采智能终端设备，C类使用简易智能终端设备，D类通过数据接口自动上传，E类通过客户手动上传。", + "expected_keywords": ["A类", "B类", "C类", "D类", "E类", "新现代", "普通现代"], + "difficulty": "easy" + }, + { + "id": "q038", + "query": "终端运营管理中有哪些禁止行为？", + "query_type": "enumeration", + "relevant_docs": ["3.docx"], + "reference_answer": "终端运营管理中规定了10条禁止行为，用于保护品牌价值和规范终端经营行为。", + "expected_keywords": ["10条", "禁止行为", "品牌价值", "经营"], + "difficulty": "medium" + }, + { + "id": "q039", + "query": "加盟终端和合作终端有什么区别？", + "query_type": "comparison", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端使用'金丝利零售'品牌，签订3年加盟协议，实行'六个统一'管理。合作终端是另一种合作模式，与加盟终端在品牌使用、管理方式、投入标准等方面有所不同。", + "expected_keywords": ["金丝利零售", "加盟协议", "合作", "品牌", "管理方式"], + "difficulty": "medium" + }, + { + "id": "q040", + "query": "终端评价不合格会怎么处理？", + "query_type": "reasoning", + "relevant_docs": ["3.docx"], + "reference_answer": "终端评价不合格将进入退出管理流程，根据评价结果进行降级或退出处理。退出管理定义了具体的降级和移除程序。", + "expected_keywords": ["退出管理", "降级", "退出", "评价", "不合格"], + "difficulty": "medium" + }, + { + "id": "q041", + "query": "直营终端有什么特殊要求？", + "query_type": "reasoning", + "relevant_docs": ["3.docx"], + "reference_answer": "直营终端是由公司直接拥有和运营的终端，在品牌形象、服务标准、运营管理等方面执行最高标准，是终端体系中的最高层级。", + "expected_keywords": ["直营", "公司", "运营", "最高", "层级"], + "difficulty": "medium" + }, + { + "id": "q042", + "query": "普通终端如何升级为现代终端？", + "query_type": "reasoning", + "relevant_docs": ["3.docx"], + "reference_answer": "普通终端可通过'五化'建设升级为五化普通终端，进而达到现代终端标准。五化是普通终端的升级路径，包括店面形象、经营管理、信息化等方面的提升。", + "expected_keywords": ["五化", "升级", "普通终端", "现代终端", "店面", "经营管理"], + "difficulty": "hard" + }, + { + "id": "q043", + "query": "终端建设标准的附件有多少个？", + "query_type": "simple_fact", + "relevant_docs": ["3.docx"], + "reference_answer": "终端梯次化建设标准共有20个附件，包括新现代终端客户申请书、现代终端合作协议、加盟终端合作协议、评价评分表、审批表、维护申请表等各类模板。", + "expected_keywords": ["20个", "附件", "申请书", "合作协议", "评价"], + "difficulty": "easy" + }, + { + "id": "q044", + "query": "三峡工程有哪些综合效益？", + "query_type": "enumeration", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "三峡工程的综合效益包括防洪、发电、航运、水资源利用、生态环境保护等。三峡电站是世界上总装机容量最大的水电站。", + "expected_keywords": ["防洪", "发电", "航运", "水资源", "生态", "综合效益"], + "difficulty": "easy" + }, + { + "id": "q045", + "query": "2022年三峡电站发电量为什么比往年低？", + "query_type": "reasoning", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年三峡电站年度发电量为787.90亿千瓦时，为2012年以来最低值。主要原因是长江流域遭遇1961年以来最严重的夏秋连旱，7-9月入库水量仅1007亿立方米。", + "expected_keywords": ["787.90", "干旱", "1961年", "夏秋连旱", "1007亿", "入库水量"], + "difficulty": "medium" + }, + { + "id": "q046", + "query": "三峡工程在防洪方面发挥了什么作用？", + "query_type": "definition", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "三峡水库蓄水运行以来，至2022年累计汛期拦洪总量2005.16亿立方米，有效应对了多次区域性大洪水，大大降低了中下游洪水位。", + "expected_keywords": ["2005.16亿", "拦洪", "汛期", "洪水位", "中下游"], + "difficulty": "medium" + }, + { + "id": "q047", + "query": "三峡船闸和升船机2022年的运行情况对比如何？", + "query_type": "comparison", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年三峡船闸运行10400闸次，通过船舶40641艘次，货运量1.56亿吨，通航率92.72%。升船机运行4470厢次，通过船舶4506艘次，旅客54416人次。", + "expected_keywords": ["10400", "40641", "1.56亿吨", "92.72%", "4470", "54416"], + "difficulty": "medium" + }, + { + "id": "q048", + "query": "三峡工程对节能减排有什么贡献？", + "query_type": "reasoning", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "截至2022年底，三峡电站累计发出清洁电力相当于节约标准煤4.85亿吨，减少二氧化碳排放12.65亿吨。", + "expected_keywords": ["4.85亿吨", "标准煤", "12.65亿吨", "二氧化碳", "清洁电力"], + "difficulty": "medium" + }, + { + "id": "q049", + "query": "2022年三峡水库的泥沙淤积情况如何？排沙比是多少？", + "query_type": "table_data", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年三峡水库入库悬移质输沙量0.136亿吨，出库0.026亿吨，库区淤积0.110亿吨，排沙比19.3%。", + "expected_keywords": ["0.136亿吨", "0.026亿吨", "0.110亿吨", "19.3%", "排沙比"], + "difficulty": "medium" + }, + { + "id": "q050", + "query": "2022年三峡水库蓄水目标完成了吗？为什么？", + "query_type": "reasoning", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年三峡水库库水位最高蓄至160.04米，未完成175米蓄水目标。原因是长江流域遭遇严重干旱，入库水量减少，同时为应对下游旱情多次补水。", + "expected_keywords": ["160.04米", "175米", "未完成", "干旱", "补水"], + "difficulty": "hard" + }, + { + "id": "q051", + "query": "2022年三峡水库为什么频繁向下游补水？补了多少？", + "query_type": "reasoning", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年长江流域遭遇严重干旱，三峡水库频繁向下游补水是为缓解下游旱情。枯水季节累计补水158天，补水总量217.76亿立方米。", + "expected_keywords": ["158天", "217.76亿", "旱情", "枯水季节", "补水"], + "difficulty": "medium" + }, + { + "id": "q052", + "query": "三峡工程2022年进行了哪些生态调度试验？", + "query_type": "enumeration", + "relevant_docs": ["三峡公报_1-15页.pdf"], + "reference_answer": "2022年进行了三类生态调度：1.针对四大家鱼等产漂流性卵鱼类繁殖的水文生态调度试验；2.针对库区产黏沉性卵鱼类自然繁殖的生态调度试验；3.库尾减淤调度试验。", + "expected_keywords": ["四大家鱼", "漂流性卵", "黏沉性卵", "减淤", "生态调度"], + "difficulty": "medium" + }, + { + "id": "q053", + "query": "三份工作规范文档分别涉及什么领域？各自适用范围是什么？", + "query_type": "cross_doc", + "relevant_docs": ["1.docx", "2.docx", "3.docx"], + "reference_answer": "1.docx是货源投放工作规范，适用于烟草产品的供应分配管理。2.docx是文明吸烟环境建设/管理/维护标准，适用于吸烟设施的建设与运维。3.docx是零售终端梯次化建设标准，适用于卷烟零售终端的分级管理。三份文档均属于江苏省烟草行业规范。", + "expected_keywords": ["货源投放", "吸烟环境", "零售终端", "江苏省", "烟草"], + "difficulty": "hard" + }, + { + "id": "q054", + "query": "三份文档中的分类体系有什么相似之处？", + "query_type": "cross_doc", + "relevant_docs": ["1.docx", "2.docx", "3.docx"], + "reference_answer": "三份文档都采用了分层分类的管理方法：货源投放将品规分为紧俏/均衡/宽松三类，客户按档级分类；吸烟环境将设施分为室/区/点三类并按A/B/C区域分类；零售终端分为普通/现代/加盟/合作/直营五个层级。", + "expected_keywords": ["分类", "分层", "品规", "设施", "终端", "层级"], + "difficulty": "hard" + }, + { + "id": "q055", + "query": "货源投放规范和终端建设标准在评价体系上有什么共同点？", + "query_type": "cross_doc", + "relevant_docs": ["1.docx", "3.docx"], + "reference_answer": "两者都采用了量化评分体系：货源投放使用双层双级指标体系评价市场状态，终端建设使用100分制评价终端达标情况。都设置了明确的评分阈值和等级划分标准。", + "expected_keywords": ["量化", "评分", "指标体系", "阈值", "等级"], + "difficulty": "hard" + }, + { + "id": "q056", + "query": "我们公司食堂在哪里？", + "query_type": "negative", + "relevant_docs": [], + "reference_answer": "知识库中没有关于公司食堂的信息。现有文档主要涉及货源投放工作规范、文明吸烟环境标准和零售终端建设标准。", + "expected_keywords": ["未找到", "未"], + "difficulty": "easy" + }, + { + "id": "q057", + "query": "公司的股票代码是什么？", + "query_type": "negative", + "relevant_docs": [], + "reference_answer": "知识库中没有关于股票代码的信息。现有文档主要涉及烟草行业的工作规范和建设标准。", + "expected_keywords": ["未找到", "未"], + "difficulty": "easy" + }, + { + "id": "q058", + "query": "如何申请年休假？需要哪些流程？", + "query_type": "negative", + "relevant_docs": [], + "reference_answer": "知识库中没有关于年休假申请流程的信息。现有文档涉及的是货源投放、吸烟环境建设和零售终端管理。", + "expected_keywords": ["未找到", "未"], + "difficulty": "easy" + }, + { + "id": "q059", + "query": "紧俏品规的上限比例是多少？", + "query_type": "paraphrase", + "relevant_docs": ["1.docx"], + "reference_answer": "紧俏品规数不得超过地市级公司所经营品规数的20%。", + "expected_keywords": ["20%", "不得超过"], + "difficulty": "easy", + "paraphrase_of": "q004" + }, + { + "id": "q060", + "query": "吸烟环境的七种功能分别是什么？", + "query_type": "paraphrase", + "relevant_docs": ["2.docx"], + "reference_answer": "SUCCESS功能框架包含7个功能：S-Smoking（吸烟）、U-Utility（实用）、C-Convenience（便利）、C-Comfortable（舒适）、E-Experience（体验）、S-Safe（安全）、S-Survey（调研）。", + "expected_keywords": ["Smoking", "Utility", "Convenience", "Comfortable", "Experience", "Safe", "Survey"], + "difficulty": "medium", + "paraphrase_of": "q018" + }, + { + "id": "q061", + "query": "加盟门店之间最少要隔多远？", + "query_type": "paraphrase", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端之间的最小间距为300米。", + "expected_keywords": ["300米", "间距"], + "difficulty": "easy", + "paraphrase_of": "q033" + }, + { + "id": "q062", + "query": "金丝利零售门店每天最少营业多长时间？", + "query_type": "paraphrase", + "relevant_docs": ["3.docx"], + "reference_answer": "加盟终端每日营业时间不低于12小时。", + "expected_keywords": ["12小时", "营业"], + "difficulty": "easy", + "paraphrase_of": "q035" + } + ], + "statistics": { + "total_queries": 62, + "by_category": { + "simple_fact": 15, + "enumeration": 13, + "reasoning": 11, + "comparison": 7, + "paraphrase": 4, + "definition": 3, + "table_data": 3, + "cross_doc": 3, + "negative": 3 + }, + "by_difficulty": { + "easy": 22, + "medium": 30, + "hard": 10 + }, + "by_document": { + "1.docx": 21, + "2.docx": 13, + "3.docx": 21, + "三峡公报_1-15页.pdf": 9, + "negative(无相关文档)": 3, + "cross_doc(多文档)": 3, + "paraphrase(改写)": 4 + } + } +}