fix(boundary): 修复多库边界问题、版本管理及删除清理
多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名 上传与版本管理修复: - 同名文件上传改为覆盖模式,自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试 文档: - 新增风险边界问题修复注意事项.md(面向后端的对接文档) - 新增向量库边界风险分析.md - 更新多篇现有文档
This commit is contained in:
@@ -215,6 +215,38 @@ class KnowledgeBaseManager(
|
||||
# 合并跨页表格
|
||||
chunks = self._merge_cross_page_tables(chunks)
|
||||
|
||||
filename = Path(filepath).name
|
||||
|
||||
# 入库前清理同名旧切片,防止重复上传导致新旧切片共存
|
||||
existing = collection.get(where={"source": filename})
|
||||
if existing and existing['ids']:
|
||||
old_count = len(existing['ids'])
|
||||
collection.delete(ids=existing['ids'])
|
||||
logger.info(f"替换模式: 清理旧切片 {filename} -> {kb_name}, 共 {old_count} 个")
|
||||
|
||||
# 清理关联的 DocStore 文件
|
||||
try:
|
||||
docstore_dir = Path('.data/docstore')
|
||||
if docstore_dir.exists():
|
||||
for ds_file in docstore_dir.glob(f'{kb_name}_{filename}_*.json'):
|
||||
ds_file.unlink()
|
||||
logger.debug(f"清理 DocStore: {ds_file.name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理 DocStore 失败: {e}")
|
||||
|
||||
# 重建 BM25 索引(移除旧条目)
|
||||
self._bm25_indexes.pop(kb_name, None)
|
||||
bm25 = self.get_bm25_index(kb_name)
|
||||
if bm25:
|
||||
remaining = collection.get(include=["documents", "metadatas"])
|
||||
if remaining['ids']:
|
||||
bm25.add_documents(
|
||||
remaining['ids'],
|
||||
remaining['documents'] or [],
|
||||
remaining['metadatas'] or []
|
||||
)
|
||||
self.save_bm25_index(kb_name)
|
||||
|
||||
# 准备向量模型
|
||||
if embedding_model is None:
|
||||
from core.engine import get_engine
|
||||
@@ -229,7 +261,6 @@ class KnowledgeBaseManager(
|
||||
metadatas = []
|
||||
embeddings = []
|
||||
|
||||
filename = Path(filepath).name
|
||||
doc_type = _get_doc_type(filename)
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
@@ -622,50 +653,73 @@ class KnowledgeBaseManager(
|
||||
def mark_document_as_superseded(
|
||||
self,
|
||||
kb_name: str,
|
||||
old_filename: str,
|
||||
new_filename: str,
|
||||
filename: str,
|
||||
new_version: str = "",
|
||||
reason: str = "版本更新"
|
||||
) -> Dict:
|
||||
"""标记文档为已替代版本"""
|
||||
collection = self.get_collection(kb_name)
|
||||
if not collection:
|
||||
return {"success": False, "error": "向量库不存在"}
|
||||
"""
|
||||
标记文档旧版本为已替代(仅更新 SQLite 版本记录)
|
||||
|
||||
result = collection.get(where={"source": old_filename})
|
||||
ChromaDB 中的旧切片由 Phase 3 去重逻辑自动清理,
|
||||
此方法只负责在 document_versions 表中将 active 版本改为 superseded。
|
||||
|
||||
if not result['ids']:
|
||||
return {"success": False, "error": "旧文档不存在"}
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
filename: 文件名
|
||||
new_version: 新版本号(如 "v2"),用于日志记录
|
||||
reason: 替代原因
|
||||
|
||||
Returns:
|
||||
{"success": True, "superseded_version": "v1"}
|
||||
"""
|
||||
from datetime import datetime
|
||||
superseded_date = datetime.now().isoformat()
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
from data.db import get_connection
|
||||
|
||||
updated_metadatas = [
|
||||
{
|
||||
**m,
|
||||
"status": "superseded",
|
||||
"superseded_by": new_filename,
|
||||
"superseded_date": superseded_date,
|
||||
"superseded_reason": reason
|
||||
vq = get_version_query()
|
||||
active = vq.get_active_version(kb_name, filename)
|
||||
if not active:
|
||||
return {"success": True, "superseded_version": None,
|
||||
"message": "无 active 版本需要标记"}
|
||||
|
||||
superseded_date = datetime.now().isoformat()
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute("""
|
||||
UPDATE document_versions
|
||||
SET status = 'superseded',
|
||||
deprecated_date = ?,
|
||||
deprecated_reason = ?
|
||||
WHERE collection = ? AND document_id = ? AND version = ?
|
||||
""", (superseded_date, reason,
|
||||
kb_name, filename, active.version))
|
||||
conn.commit()
|
||||
|
||||
# 记录变更日志
|
||||
vq.log_version_change(
|
||||
kb_name, filename,
|
||||
change_type="supersede",
|
||||
old_version=active.version,
|
||||
new_version=new_version,
|
||||
old_status="active",
|
||||
new_status="superseded",
|
||||
reason=reason
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"标记版本替代: {kb_name}/{filename} "
|
||||
f"{active.version} -> {new_version or '(待创建)'}"
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"superseded_version": active.version,
|
||||
"new_version": new_version,
|
||||
"collection": kb_name
|
||||
}
|
||||
for m in result['metadatas']
|
||||
]
|
||||
|
||||
collection.update(
|
||||
ids=result['ids'],
|
||||
metadatas=updated_metadatas
|
||||
)
|
||||
|
||||
self.rebuild_bm25_index(kb_name)
|
||||
|
||||
logger.info(f"标记文档替代: {old_filename} -> {new_filename}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"superseded_chunks": len(result['ids']),
|
||||
"old_document": old_filename,
|
||||
"new_document": new_filename,
|
||||
"collection": kb_name
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"标记替代版本失败: {e}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
# ==================== 辅助检索方法 ====================
|
||||
|
||||
|
||||
Reference in New Issue
Block a user