fix(boundary): 修复多库边界问题、版本管理及删除清理
多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名 上传与版本管理修复: - 同名文件上传改为覆盖模式,自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试 文档: - 新增风险边界问题修复注意事项.md(面向后端的对接文档) - 新增向量库边界风险分析.md - 更新多篇现有文档
This commit is contained in:
@@ -415,12 +415,16 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
||||
if not contexts:
|
||||
return {"answer_with_refs": answer, "citations": []}
|
||||
|
||||
# 按 chunk_id 组织 contexts
|
||||
# 按 (collection, chunk_id) 复合键组织 contexts,防止跨库同名文件覆盖
|
||||
ctx_by_chunk = {}
|
||||
for ctx in contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
chunk_id = meta.get('chunk_id') or f"{meta.get('source')}_{meta.get('chunk_index', 0)}"
|
||||
ctx_by_chunk[chunk_id] = ctx
|
||||
coll = meta.get('_collection') or meta.get('collection') or ''
|
||||
composite_key = f"{coll}/{chunk_id}" if coll else chunk_id
|
||||
# 保存原始 chunk_id,用于对外输出(ref tag / citation)
|
||||
ctx['_raw_chunk_id'] = chunk_id
|
||||
ctx_by_chunk[composite_key] = ctx
|
||||
|
||||
# jieba 分词函数(fallback 到字符级)
|
||||
try:
|
||||
@@ -485,20 +489,27 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
||||
if cid not in cited_set:
|
||||
cited_set.add(cid)
|
||||
cited_chunks_ordered.append(cid)
|
||||
# 在段落末尾插入引用标记(多个引用连续排列)
|
||||
ref_tags = "".join(f"[ref:{cid}]" for cid in selected_ids)
|
||||
# 在段落末尾插入引用标记(使用原始 chunk_id,不暴露复合键)
|
||||
ref_tags = "".join(
|
||||
f"[ref:{ctx_by_chunk[cid].get('_raw_chunk_id', cid)}]"
|
||||
for cid in selected_ids
|
||||
)
|
||||
result_parts.append(f"{para}{ref_tags}{sep}")
|
||||
else:
|
||||
result_parts.append(para + sep)
|
||||
|
||||
# 构建引用列表(按出现顺序)
|
||||
# 构建引用列表(按出现顺序),使用原始 chunk_id 构建 citation
|
||||
citations = []
|
||||
for chunk_id in cited_chunks_ordered:
|
||||
ctx = ctx_by_chunk.get(chunk_id)
|
||||
for composite_key in cited_chunks_ordered:
|
||||
ctx = ctx_by_chunk.get(composite_key)
|
||||
if ctx:
|
||||
meta = ctx.get('meta', {})
|
||||
full_content = ctx.get('doc', '')
|
||||
citation = _build_citation(meta, full_content)
|
||||
# 确保 citation 中的 chunk_id 使用原始值(不含 collection 前缀)
|
||||
raw_id = ctx.get('_raw_chunk_id')
|
||||
if raw_id:
|
||||
citation['chunk_id'] = raw_id
|
||||
citations.append(citation)
|
||||
|
||||
return {
|
||||
@@ -587,7 +598,7 @@ def _build_citation(meta: Dict, full_content: str = '') -> Dict[str, Any]:
|
||||
"chunk_id": chunk_id_raw,
|
||||
"chunk_index": chunk_index, # 全局切片序号,用于精准定位文档位置
|
||||
"source": meta.get('source', ''),
|
||||
"collection": meta.get('_collection', ''), # 所属向量库,用于前端文档预览跳转
|
||||
"collection": meta.get('_collection') or meta.get('collection', ''), # 所属向量库,用于前端文档预览跳转
|
||||
"doc_type": meta.get('doc_type', 'other'),
|
||||
"section": _clean_section(meta.get('section', '')),
|
||||
"preview": meta.get('preview', ''),
|
||||
@@ -1528,12 +1539,8 @@ def rag():
|
||||
meta['_retrieval_rank'] = rank
|
||||
# 确保 _collection 字段存在(单知识库路径下 ChromaDB 原生不返回此字段)
|
||||
if not meta.get('_collection'):
|
||||
if collections and len(collections) == 1:
|
||||
meta['_collection'] = collections[0]
|
||||
elif collections:
|
||||
meta['_collection'] = collections[0] # 多知识库时回退到第一个
|
||||
else:
|
||||
meta['_collection'] = 'public_kb'
|
||||
# 优先使用入库时写入的 collection 字段
|
||||
meta['_collection'] = meta.get('collection') or (collections[0] if collections else 'public_kb')
|
||||
source_name = meta.get('source', '未知')
|
||||
if source_name not in seen_sources or score > seen_sources[source_name]['score']:
|
||||
doc_type = meta.get('doc_type', 'other')
|
||||
|
||||
Reference in New Issue
Block a user