diff --git a/core/mmr.py b/core/mmr.py index 7e8c29e..4565fd1 100644 --- a/core/mmr.py +++ b/core/mmr.py @@ -108,22 +108,44 @@ def mmr_rerank( return selected +def _tokenize_words(text: str) -> set: + """ + 使用 jieba 分词并过滤噪声,返回有意义的词集合。 + + 过滤规则: + - 去除单字符词(如 "的", "了", "在")—— 这些是停用词,对区分文档无意义 + - 去除纯数字 / 纯标点 + - 保留 2 字及以上的实词 + """ + import jieba + words = set() + for w in jieba.cut(text): + w = w.strip() + if len(w) >= 2 and not w.isdigit(): + words.add(w) + return words + + def mmr_filter_by_content( candidates: List[Dict], top_k: int = 30, - similarity_threshold: float = 0.9 + similarity_threshold: float = 0.85 ) -> List[Dict]: """ - 基于内容相似度的去重(简化版,不需要 embedding) + 基于 jieba 词级 Jaccard 相似度的去重(不需要 embedding) + + 与旧版字符级 set(text) 的区别: + - 旧版:set("安全生产管理制度") → {'安','全','生','产',...},中文文档间字符集合高度重叠 + - 新版:jieba 分词 → {"安全生产", "管理制度", ...},词级集合区分度高 适用于: - - 没有 embedding 的情况 - - 快速去重场景 + - MMR_USE_EMBEDDING=False 时的快速去重 + - 避免 CPU 编码 100+ 文档的 50 秒开销 Args: candidates: 候选文档列表 top_k: 返回数量 - similarity_threshold: 相似度阈值,超过则视为重复 + similarity_threshold: 相似度阈值,超过则视为重复(默认 0.85) Returns: 去重后的候选文档列表 @@ -134,35 +156,42 @@ def mmr_filter_by_content( if len(candidates) <= top_k: return candidates - selected = [] - remaining = candidates.copy() + # 预分词:对所有候选文档一次性分词,避免重复调用 jieba.cut + word_sets = [] + for c in candidates: + content = c.get('content', c.get('document', ''))[:500] + word_sets.append(_tokenize_words(content)) - while len(selected) < top_k and remaining: - current = remaining.pop(0) + selected_indices = [] + + for i in range(len(candidates)): + if len(selected_indices) >= top_k: + break + + current_words = word_sets[i] + if not current_words: + # 空内容直接保留 + selected_indices.append(i) + continue - # 检查是否与已选内容重复 is_duplicate = False - current_content = current.get('content', current.get('document', ''))[:200] + for j in selected_indices: + selected_words = word_sets[j] + if not selected_words: + continue - for s in selected: - s_content = s.get('content', s.get('document', ''))[:200] + intersection = len(current_words & selected_words) + union = len(current_words | selected_words) + similarity = intersection / union if union > 0 else 0 - # 简单的 Jaccard 相似度 - words1 = set(current_content) - words2 = set(s_content) - if words1 and words2: - intersection = len(words1 & words2) - union = len(words1 | words2) - similarity = intersection / union if union > 0 else 0 - - if similarity > similarity_threshold: - is_duplicate = True - break + if similarity > similarity_threshold: + is_duplicate = True + break if not is_duplicate: - selected.append(current) + selected_indices.append(i) - return selected + return [candidates[i] for i in selected_indices] # ==================== 测试 ====================