fix(boundary): 修复多库边界问题、版本管理及删除清理
多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名 上传与版本管理修复: - 同名文件上传改为覆盖模式,自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试 文档: - 新增风险边界问题修复注意事项.md(面向后端的对接文档) - 新增向量库边界风险分析.md - 更新多篇现有文档
This commit is contained in:
@@ -43,7 +43,7 @@ class DocumentMixin:
|
||||
]
|
||||
|
||||
def delete_document(self, kb_name: str, filename: str) -> int:
|
||||
"""从向量库删除文档"""
|
||||
"""从向量库删除文档,并清理 SQLite 版本记录"""
|
||||
collection = self.get_collection(kb_name)
|
||||
if not collection:
|
||||
return 0
|
||||
@@ -56,6 +56,22 @@ class DocumentMixin:
|
||||
collection.delete(ids=result['ids'])
|
||||
deleted = len(result['ids'])
|
||||
|
||||
# 清理 SQLite 版本记录和变更日志
|
||||
try:
|
||||
from data.db import get_connection
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM document_versions WHERE collection = ? AND document_id = ?",
|
||||
(kb_name, filename)
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM version_change_logs WHERE collection = ? AND document_id = ?",
|
||||
(kb_name, filename)
|
||||
)
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"清理版本记录失败: {e}")
|
||||
|
||||
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
||||
return deleted
|
||||
|
||||
@@ -66,7 +82,7 @@ class DocumentMixin:
|
||||
reason: str = "制度废止",
|
||||
deprecated_by: str = ""
|
||||
) -> Dict:
|
||||
"""软删除文档 - 将chunks状态标记为deprecated"""
|
||||
"""软删除文档 - 将chunks状态标记为deprecated,并同步 SQLite 版本记录"""
|
||||
collection = self.get_collection(kb_name)
|
||||
if not collection:
|
||||
return {"success": False, "error": "向量库不存在"}
|
||||
@@ -97,6 +113,30 @@ class DocumentMixin:
|
||||
|
||||
logger.info(f"软删除文档: {kb_name}/{filename}, chunks: {len(result['ids'])}, 原因: {reason}")
|
||||
|
||||
# 同步 SQLite 版本记录
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
from data.db import get_connection
|
||||
vq = get_version_query()
|
||||
active = vq.get_active_version(kb_name, filename)
|
||||
if active:
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute("""
|
||||
UPDATE document_versions
|
||||
SET status='deprecated', deprecated_date=?, deprecated_reason=?
|
||||
WHERE collection=? AND document_id=? AND version=?
|
||||
""", (deprecated_date, reason, kb_name, filename, active.version))
|
||||
conn.commit()
|
||||
vq.log_version_change(
|
||||
kb_name, filename,
|
||||
change_type="deprecate",
|
||||
old_version=active.version,
|
||||
old_status="active", new_status="deprecated",
|
||||
reason=reason, changed_by=deprecated_by
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"同步版本记录失败: {e}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"deprecated_chunks": len(result['ids']),
|
||||
@@ -106,7 +146,7 @@ class DocumentMixin:
|
||||
}
|
||||
|
||||
def restore_document(self, kb_name: str, filename: str) -> Dict:
|
||||
"""恢复已废止的文档"""
|
||||
"""恢复已废止的文档,并同步 SQLite 版本记录"""
|
||||
collection = self.get_collection(kb_name)
|
||||
if not collection:
|
||||
return {"success": False, "error": "向量库不存在"}
|
||||
@@ -142,6 +182,32 @@ class DocumentMixin:
|
||||
|
||||
logger.info(f"恢复文档: {kb_name}/{filename}, chunks: {len(result['ids'])}")
|
||||
|
||||
# 同步 SQLite 版本记录
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
from data.db import get_connection
|
||||
vq = get_version_query()
|
||||
history = vq.get_document_history(kb_name, filename)
|
||||
deprecated_ver = next(
|
||||
(v for v in history if v.status.value == 'deprecated'), None
|
||||
)
|
||||
if deprecated_ver:
|
||||
with get_connection("knowledge") as conn:
|
||||
conn.execute("""
|
||||
UPDATE document_versions
|
||||
SET status='active', deprecated_date=NULL, deprecated_reason=NULL
|
||||
WHERE collection=? AND document_id=? AND version=?
|
||||
""", (kb_name, filename, deprecated_ver.version))
|
||||
conn.commit()
|
||||
vq.log_version_change(
|
||||
kb_name, filename,
|
||||
change_type="restore",
|
||||
old_version=deprecated_ver.version,
|
||||
old_status="deprecated", new_status="active"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"同步版本记录失败: {e}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"restored_chunks": len(result['ids']),
|
||||
|
||||
Reference in New Issue
Block a user