fix(boundary): 修复多库边界问题、版本管理及删除清理
多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名 上传与版本管理修复: - 同名文件上传改为覆盖模式,自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试 文档: - 新增风险边界问题修复注意事项.md(面向后端的对接文档) - 新增向量库边界风险分析.md - 更新多篇现有文档
This commit is contained in:
@@ -243,14 +243,75 @@ def upload_document() -> Tuple[Any, int]:
|
||||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||||
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
if os.path.exists(filepath):
|
||||
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
|
||||
name, ext_part = os.path.splitext(filename)
|
||||
filename = f"{name}{timestamp}{ext_part}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
|
||||
# 同名文件处理:清理旧切片 + 覆盖旧文件(替代原来的时间戳重命名策略)
|
||||
replaced = False
|
||||
if os.path.exists(filepath):
|
||||
replaced = True
|
||||
# 1. 清理旧切片(ChromaDB + BM25 + DocStore)
|
||||
kb_manager = _get_kb_manager()
|
||||
if kb_manager:
|
||||
old_chunks = kb_manager.get_document_chunks(collection, filename)
|
||||
if old_chunks:
|
||||
old_ids = [c['id'] for c in old_chunks]
|
||||
coll_obj = kb_manager.get_collection(collection)
|
||||
if coll_obj:
|
||||
coll_obj.delete(ids=old_ids)
|
||||
logger.info(f"替换上传: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
|
||||
|
||||
# 清理关联的 DocStore 文件
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
docstore_dir = _Path('.data/docstore')
|
||||
if docstore_dir.exists():
|
||||
for ds_file in docstore_dir.glob(f'{collection}_{filename}_*.json'):
|
||||
ds_file.unlink()
|
||||
except Exception as e:
|
||||
logger.warning(f"清理 DocStore 失败: {e}")
|
||||
|
||||
# 重建 BM25 索引
|
||||
kb_manager.rebuild_bm25_index(collection)
|
||||
|
||||
# 2. 清理同步哈希记录(以便 sync 能正确检测为新文件)
|
||||
try:
|
||||
from knowledge.sync import SyncDatabase
|
||||
sync_db = SyncDatabase()
|
||||
sync_db.delete_document_hash(f"{collection}/{filename}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理哈希记录失败: {e}")
|
||||
|
||||
# 保存文件(覆盖同名旧文件)
|
||||
file.save(filepath)
|
||||
|
||||
# 版本管理:覆盖上传时标记旧版本为 superseded(新版本记录由 sync 服务统一创建)
|
||||
if replaced:
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
from data.db import get_connection
|
||||
vq = get_version_query()
|
||||
active = vq.get_active_version(collection, filename)
|
||||
|
||||
if active:
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute("""
|
||||
UPDATE document_versions
|
||||
SET status='superseded', deprecated_date=?,
|
||||
deprecated_reason='重新上传覆盖'
|
||||
WHERE collection=? AND document_id=? AND version=?
|
||||
""", (datetime.now().isoformat(), collection,
|
||||
filename, active.version))
|
||||
conn.commit()
|
||||
vq.log_version_change(
|
||||
collection, filename,
|
||||
change_type="reupload",
|
||||
old_version=active.version,
|
||||
old_status="active", new_status="superseded",
|
||||
reason="重新上传覆盖",
|
||||
changed_by=request.current_user.get('user_id', '')
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"标记旧版本失败: {e}")
|
||||
|
||||
# 6. 触发向量化
|
||||
sync_status = "已保存,等待手动同步"
|
||||
sync_service = _get_sync_service()
|
||||
@@ -276,7 +337,8 @@ def upload_document() -> Tuple[Any, int]:
|
||||
"filename": filename,
|
||||
"collection": collection,
|
||||
"path": f"{target_subdir}/{filename}",
|
||||
"size": file_size
|
||||
"size": file_size,
|
||||
"replaced": replaced
|
||||
},
|
||||
"sync_status": sync_status
|
||||
},
|
||||
@@ -350,19 +412,56 @@ def batch_upload_documents() -> Tuple[Any, int]:
|
||||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
|
||||
# 处理重名
|
||||
# 同名文件处理:清理旧切片 + 覆盖旧文件
|
||||
replaced = False
|
||||
if os.path.exists(filepath):
|
||||
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
|
||||
name, ext_part = os.path.splitext(filename)
|
||||
filename = f"{name}{timestamp}{ext_part}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
replaced = True
|
||||
kb_mgr = _get_kb_manager()
|
||||
if kb_mgr:
|
||||
old_chunks = kb_mgr.get_document_chunks(collection, filename)
|
||||
if old_chunks:
|
||||
old_ids = [c['id'] for c in old_chunks]
|
||||
coll_obj = kb_mgr.get_collection(collection)
|
||||
if coll_obj:
|
||||
coll_obj.delete(ids=old_ids)
|
||||
logger.info(f"批量替换: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
|
||||
kb_mgr.rebuild_bm25_index(collection)
|
||||
|
||||
try:
|
||||
from knowledge.sync import SyncDatabase
|
||||
sync_db = SyncDatabase()
|
||||
sync_db.delete_document_hash(f"{collection}/{filename}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理哈希记录失败: {e}")
|
||||
|
||||
file.save(filepath)
|
||||
|
||||
# 版本管理:覆盖上传时标记旧版本为 superseded(新版本记录由 sync 服务统一创建)
|
||||
if replaced:
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
from data.db import get_connection
|
||||
vq = get_version_query()
|
||||
active = vq.get_active_version(collection, filename)
|
||||
|
||||
if active:
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute("""
|
||||
UPDATE document_versions
|
||||
SET status='superseded', deprecated_date=?,
|
||||
deprecated_reason='批量上传覆盖'
|
||||
WHERE collection=? AND document_id=? AND version=?
|
||||
""", (datetime.now().isoformat(), collection,
|
||||
filename, active.version))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"批量上传标记旧版本失败: {e}")
|
||||
|
||||
results.append({
|
||||
"filename": filename,
|
||||
"status": "success",
|
||||
"path": f"{target_subdir}/{filename}"
|
||||
"path": f"{target_subdir}/{filename}",
|
||||
"replaced": replaced
|
||||
})
|
||||
except Exception as e:
|
||||
results.append({
|
||||
|
||||
Reference in New Issue
Block a user