fix(boundary): 修复多库边界问题、版本管理及删除清理
多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键,修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀,修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名 上传与版本管理修复: - 同名文件上传改为覆盖模式,自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试 文档: - 新增风险边界问题修复注意事项.md(面向后端的对接文档) - 新增向量库边界风险分析.md - 更新多篇现有文档
This commit is contained in:
@@ -19,69 +19,52 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def cleanup_superseded_versions(days_to_keep: int = 7) -> int:
|
||||
"""
|
||||
清理超过指定天数的 superseded 版本
|
||||
清理超过保留期的 superseded 版本记录(SQLite)
|
||||
|
||||
按新设计,superseded 切片在 ChromaDB 中已不存在(由 Phase 3 去重删除),
|
||||
此函数只清理 SQLite 中的旧版本记录和变更日志。
|
||||
|
||||
Args:
|
||||
days_to_keep: 保留天数,默认7天
|
||||
|
||||
Returns:
|
||||
清理的 chunk 数量
|
||||
清理的版本记录数量
|
||||
"""
|
||||
from knowledge.manager import get_kb_manager
|
||||
|
||||
kb_manager = get_kb_manager()
|
||||
cutoff_date = (datetime.now() - timedelta(days=days_to_keep)).isoformat()
|
||||
|
||||
logger.info(f"开始清理 superseded 版本(保留 {days_to_keep} 天内的)")
|
||||
logger.info(f"开始清理 superseded 版本记录(保留 {days_to_keep} 天内的)")
|
||||
|
||||
# 获取所有向量库
|
||||
try:
|
||||
kb_names = kb_manager.list_collections()
|
||||
from data.db import get_connection
|
||||
|
||||
with get_connection("knowledge") as conn:
|
||||
# 清理超期的 superseded 版本记录
|
||||
cursor = conn.execute("""
|
||||
DELETE FROM document_versions
|
||||
WHERE status = 'superseded' AND deprecated_date < ?
|
||||
""", (cutoff_date,))
|
||||
cleaned_versions = cursor.rowcount
|
||||
|
||||
# 清理超期的变更日志
|
||||
cursor2 = conn.execute("""
|
||||
DELETE FROM version_change_logs
|
||||
WHERE created_at < ?
|
||||
""", (cutoff_date,))
|
||||
cleaned_logs = cursor2.rowcount
|
||||
|
||||
conn.commit()
|
||||
|
||||
total_cleaned = cleaned_versions + cleaned_logs
|
||||
logger.info(
|
||||
f"清理完成: {cleaned_versions} 条版本记录, "
|
||||
f"{cleaned_logs} 条变更日志"
|
||||
)
|
||||
return cleaned_versions
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取向量库列表失败: {e}")
|
||||
logger.error(f"清理 superseded 版本记录失败: {e}")
|
||||
return 0
|
||||
|
||||
total_cleaned = 0
|
||||
|
||||
for kb_name in kb_names:
|
||||
try:
|
||||
collection = kb_manager.get_collection(kb_name)
|
||||
if not collection:
|
||||
continue
|
||||
|
||||
# 查询超过保留期的 superseded chunks
|
||||
# 注意:ChromaDB 的 where 过滤可能不支持 $lt 操作符
|
||||
# 所以我们先获取所有 superseded chunks,然后在 Python 中过滤
|
||||
result = collection.get(
|
||||
where={"status": "superseded"}
|
||||
)
|
||||
|
||||
if not result['ids']:
|
||||
continue
|
||||
|
||||
# 在 Python 中过滤超过保留期的 chunks
|
||||
ids_to_delete = []
|
||||
for i, meta in enumerate(result['metadatas']):
|
||||
superseded_time = meta.get('superseded_time', '')
|
||||
if superseded_time and superseded_time < cutoff_date:
|
||||
ids_to_delete.append(result['ids'][i])
|
||||
|
||||
if ids_to_delete:
|
||||
# 删除这些 chunks
|
||||
collection.delete(ids=ids_to_delete)
|
||||
total_cleaned += len(ids_to_delete)
|
||||
logger.info(f"清理 {kb_name}: {len(ids_to_delete)} chunks")
|
||||
|
||||
# 重建 BM25 索引
|
||||
kb_manager.rebuild_bm25_index(kb_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理 {kb_name} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"清理完成,共删除 {total_cleaned} 个 superseded chunks")
|
||||
return total_cleaned
|
||||
|
||||
|
||||
def cleanup_deprecated_versions(days_to_keep: int = 30) -> int:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user