fix(boundary): 修复多库边界问题、版本管理及删除清理

多库检索与存储修复:
- RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞
- DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖
- search_multiple 去重改用复合键
- chunk_id 解析改用 rsplit 兼容下划线文件名

上传与版本管理修复:
- 同名文件上传改为覆盖模式,自动清理旧切片
- 修复首次上传不创建版本记录
- 修复覆盖上传版本号回退到 v1
- sync ADDED 分支改用动态版本号生成
- _generate_version_id 改为基于全部版本递增
- 废止/恢复操作同步 SQLite 版本记录
- mark_document_as_superseded 改为仅更新 SQLite

删除清理修复:
- 删除文档时同步清理 SQLite 版本记录和变更日志
- 删除向量库时同步清理该库所有版本记录
- cleanup 改为清理 SQLite 记录而非 ChromaDB

测试:
- test_version_management.py: 27 条版本管理单元测试
- test_edge_cases.py: 28 条边界用例测试
- test_upload_dedup.py: 5 条上传去重测试
- e2e_risk_test.py: 27 条端到端风险测试

文档:
- 新增风险边界问题修复注意事项.md(面向后端的对接文档)
- 新增向量库边界风险分析.md
- 更新多篇现有文档
This commit is contained in:
lacerate551
2026-06-04 23:58:44 +08:00
parent a1a0814633
commit cb75b9b274
50 changed files with 6385 additions and 6248 deletions

View File

@@ -1112,6 +1112,13 @@ class RAGEngine:
):
items.append((doc_id, doc, meta, dist))
existing_ids = {item[0] for item in items}
# 兼容 ID 前缀:同时存储原始 ID 和去前缀版本,确保邻居匹配不遗漏
_raw_id_set = set()
for _eid in existing_ids:
if '/' in _eid:
_raw_id_set.add(_eid.split('/', 1)[1])
else:
_raw_id_set.add(_eid)
seeds = [
(doc_id, doc, meta, dist)
@@ -1178,7 +1185,8 @@ class RAGEngine:
break
if seed_neighbors_added >= MAX_EXPANDED_NEIGHBORS:
break
if n_id in existing_ids:
# 兼容前缀 ID用原始 ID 和去前缀版本双重匹配
if n_id in existing_ids or n_id in _raw_id_set:
continue
n_meta = dict(n_meta or {})
if seed_meta.get('_collection') and not n_meta.get('_collection'):
@@ -1188,6 +1196,7 @@ class RAGEngine:
distance = seed_dist + 0.0001 * abs(n_index - seed_index)
items.append((n_id, n_doc, n_meta, distance))
existing_ids.add(n_id)
_raw_id_set.add(n_id.split('/', 1)[1] if '/' in n_id else n_id)
added += 1
seed_neighbors_added += 1
@@ -1830,6 +1839,7 @@ class RAGEngine:
if weights is None:
weights = [1.0] * len(results_list)
# 使用 (collection, doc_id) 复合键去重,防止跨库同名文件的结果被吞
doc_scores = {}
for results, weight in zip(results_list, weights):
if not results['documents'] or not results['documents'][0]:
@@ -1838,13 +1848,27 @@ class RAGEngine:
results['ids'][0], results['documents'][0], results['metadatas'][0]
)):
rrf_score = weight / (k + rank + 1)
if doc_id not in doc_scores:
doc_scores[doc_id] = {'score': 0.0, 'doc': doc, 'meta': meta}
doc_scores[doc_id]['score'] += rrf_score
coll = meta.get('_collection') or meta.get('collection') or ''
# 复合键:同 collection 内同名 doc_id 合并分数(向量+BM25跨 collection 不合并
composite_key = f"{coll}\x00{doc_id}" if coll else doc_id
if composite_key not in doc_scores:
doc_scores[composite_key] = {'score': 0.0, 'doc': doc, 'meta': meta, 'coll': coll, 'raw_id': doc_id}
doc_scores[composite_key]['score'] += rrf_score
sorted_items = sorted(doc_scores.items(), key=lambda x: x[1]['score'], reverse=True)
# 输出 ID 加 collection 前缀,确保跨库同名切片在全链路中可区分
out_ids = []
for item in sorted_items:
coll = item[1]['coll']
raw_id = item[1]['raw_id']
if coll and not raw_id.startswith(f"{coll}/"):
out_ids.append(f"{coll}/{raw_id}")
else:
out_ids.append(raw_id)
return {
'ids': [[item[0] for item in sorted_items]],
'ids': [out_ids],
'documents': [[item[1]['doc'] for item in sorted_items]],
'metadatas': [[item[1]['meta'] for item in sorted_items]],
'distances': [[item[1]['score'] for item in sorted_items]],