fix(boundary): 修复多库边界问题、版本管理及删除清理

多库检索与存储修复:
- RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞
- DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖
- search_multiple 去重改用复合键
- chunk_id 解析改用 rsplit 兼容下划线文件名

上传与版本管理修复:
- 同名文件上传改为覆盖模式,自动清理旧切片
- 修复首次上传不创建版本记录
- 修复覆盖上传版本号回退到 v1
- sync ADDED 分支改用动态版本号生成
- _generate_version_id 改为基于全部版本递增
- 废止/恢复操作同步 SQLite 版本记录
- mark_document_as_superseded 改为仅更新 SQLite

删除清理修复:
- 删除文档时同步清理 SQLite 版本记录和变更日志
- 删除向量库时同步清理该库所有版本记录
- cleanup 改为清理 SQLite 记录而非 ChromaDB

测试:
- test_version_management.py: 27 条版本管理单元测试
- test_edge_cases.py: 28 条边界用例测试
- test_upload_dedup.py: 5 条上传去重测试
- e2e_risk_test.py: 27 条端到端风险测试

文档:
- 新增风险边界问题修复注意事项.md(面向后端的对接文档)
- 新增向量库边界风险分析.md
- 更新多篇现有文档
This commit is contained in:
lacerate551
2026-06-04 23:58:44 +08:00
parent a1a0814633
commit cb75b9b274
50 changed files with 6385 additions and 6248 deletions

View File

@@ -243,14 +243,75 @@ def upload_document() -> Tuple[Any, int]:
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
filepath = os.path.join(target_dir, filename)
if os.path.exists(filepath):
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
name, ext_part = os.path.splitext(filename)
filename = f"{name}{timestamp}{ext_part}"
filepath = os.path.join(target_dir, filename)
# 同名文件处理:清理旧切片 + 覆盖旧文件(替代原来的时间戳重命名策略)
replaced = False
if os.path.exists(filepath):
replaced = True
# 1. 清理旧切片ChromaDB + BM25 + DocStore
kb_manager = _get_kb_manager()
if kb_manager:
old_chunks = kb_manager.get_document_chunks(collection, filename)
if old_chunks:
old_ids = [c['id'] for c in old_chunks]
coll_obj = kb_manager.get_collection(collection)
if coll_obj:
coll_obj.delete(ids=old_ids)
logger.info(f"替换上传: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)}")
# 清理关联的 DocStore 文件
try:
from pathlib import Path as _Path
docstore_dir = _Path('.data/docstore')
if docstore_dir.exists():
for ds_file in docstore_dir.glob(f'{collection}_{filename}_*.json'):
ds_file.unlink()
except Exception as e:
logger.warning(f"清理 DocStore 失败: {e}")
# 重建 BM25 索引
kb_manager.rebuild_bm25_index(collection)
# 2. 清理同步哈希记录(以便 sync 能正确检测为新文件)
try:
from knowledge.sync import SyncDatabase
sync_db = SyncDatabase()
sync_db.delete_document_hash(f"{collection}/{filename}")
except Exception as e:
logger.warning(f"清理哈希记录失败: {e}")
# 保存文件(覆盖同名旧文件)
file.save(filepath)
# 版本管理:覆盖上传时标记旧版本为 superseded新版本记录由 sync 服务统一创建)
if replaced:
try:
from knowledge.document_versions import get_version_query
from data.db import get_connection
vq = get_version_query()
active = vq.get_active_version(collection, filename)
if active:
with get_connection("knowledge") as conn:
conn.execute("""
UPDATE document_versions
SET status='superseded', deprecated_date=?,
deprecated_reason='重新上传覆盖'
WHERE collection=? AND document_id=? AND version=?
""", (datetime.now().isoformat(), collection,
filename, active.version))
conn.commit()
vq.log_version_change(
collection, filename,
change_type="reupload",
old_version=active.version,
old_status="active", new_status="superseded",
reason="重新上传覆盖",
changed_by=request.current_user.get('user_id', '')
)
except Exception as e:
logger.warning(f"标记旧版本失败: {e}")
# 6. 触发向量化
sync_status = "已保存,等待手动同步"
sync_service = _get_sync_service()
@@ -276,7 +337,8 @@ def upload_document() -> Tuple[Any, int]:
"filename": filename,
"collection": collection,
"path": f"{target_subdir}/{filename}",
"size": file_size
"size": file_size,
"replaced": replaced
},
"sync_status": sync_status
},
@@ -350,19 +412,56 @@ def batch_upload_documents() -> Tuple[Any, int]:
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
filepath = os.path.join(target_dir, filename)
# 处理重名
# 同名文件处理:清理旧切片 + 覆盖旧文件
replaced = False
if os.path.exists(filepath):
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
name, ext_part = os.path.splitext(filename)
filename = f"{name}{timestamp}{ext_part}"
filepath = os.path.join(target_dir, filename)
replaced = True
kb_mgr = _get_kb_manager()
if kb_mgr:
old_chunks = kb_mgr.get_document_chunks(collection, filename)
if old_chunks:
old_ids = [c['id'] for c in old_chunks]
coll_obj = kb_mgr.get_collection(collection)
if coll_obj:
coll_obj.delete(ids=old_ids)
logger.info(f"批量替换: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)}")
kb_mgr.rebuild_bm25_index(collection)
try:
from knowledge.sync import SyncDatabase
sync_db = SyncDatabase()
sync_db.delete_document_hash(f"{collection}/{filename}")
except Exception as e:
logger.warning(f"清理哈希记录失败: {e}")
file.save(filepath)
# 版本管理:覆盖上传时标记旧版本为 superseded新版本记录由 sync 服务统一创建)
if replaced:
try:
from knowledge.document_versions import get_version_query
from data.db import get_connection
vq = get_version_query()
active = vq.get_active_version(collection, filename)
if active:
with get_connection("knowledge") as conn:
conn.execute("""
UPDATE document_versions
SET status='superseded', deprecated_date=?,
deprecated_reason='批量上传覆盖'
WHERE collection=? AND document_id=? AND version=?
""", (datetime.now().isoformat(), collection,
filename, active.version))
conn.commit()
except Exception as e:
logger.warning(f"批量上传标记旧版本失败: {e}")
results.append({
"filename": filename,
"status": "success",
"path": f"{target_subdir}/{filename}"
"path": f"{target_subdir}/{filename}",
"replaced": replaced
})
except Exception as e:
results.append({