fix(boundary): 修复多库边界问题、版本管理及删除清理

多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键，修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀，修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名上传与版本管理修复: - 同名文件上传改为覆盖模式，自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试文档: - 新增风险边界问题修复注意事项.md（面向后端的对接文档） - 新增向量库边界风险分析.md - 更新多篇现有文档
2026-06-04 23:58:44 +08:00
parent a1a0814633
commit cb75b9b274
50 changed files with 6385 additions and 6248 deletions
--- a/knowledge/cleanup.py
+++ b/knowledge/cleanup.py
@@ -19,69 +19,52 @@ logger = logging.getLogger(__name__)

 def cleanup_superseded_versions(days_to_keep: int = 7) -> int:
    """
-    清理超过指定天数的 superseded 版本
+    清理超过保留期的 superseded 版本记录（SQLite）
+
+    按新设计，superseded 切片在 ChromaDB 中已不存在（由 Phase 3 去重删除），
+    此函数只清理 SQLite 中的旧版本记录和变更日志。

    Args:
        days_to_keep: 保留天数，默认7天

    Returns:
-        清理的 chunk 数量
+        清理的版本记录数量
    """
-    from knowledge.manager import get_kb_manager
-
-    kb_manager = get_kb_manager()
    cutoff_date = (datetime.now() - timedelta(days=days_to_keep)).isoformat()

-    logger.info(f"开始清理 superseded 版本（保留 {days_to_keep} 天内的）")
+    logger.info(f"开始清理 superseded 版本记录（保留 {days_to_keep} 天内的）")

-    # 获取所有向量库
    try:
-        kb_names = kb_manager.list_collections()
+        from data.db import get_connection
+
+        with get_connection("knowledge") as conn:
+            # 清理超期的 superseded 版本记录
+            cursor = conn.execute("""
+                DELETE FROM document_versions
+                WHERE status = 'superseded' AND deprecated_date < ?
+            """, (cutoff_date,))
+            cleaned_versions = cursor.rowcount
+
+            # 清理超期的变更日志
+            cursor2 = conn.execute("""
+                DELETE FROM version_change_logs
+                WHERE created_at < ?
+            """, (cutoff_date,))
+            cleaned_logs = cursor2.rowcount
+
+            conn.commit()
+
+        total_cleaned = cleaned_versions + cleaned_logs
+        logger.info(
+            f"清理完成: {cleaned_versions} 条版本记录, "
+            f"{cleaned_logs} 条变更日志"
+        )
+        return cleaned_versions
+
    except Exception as e:
-        logger.error(f"获取向量库列表失败: {e}")
+        logger.error(f"清理 superseded 版本记录失败: {e}")
        return 0

-    total_cleaned = 0
-
-    for kb_name in kb_names:
-        try:
-            collection = kb_manager.get_collection(kb_name)
-            if not collection:
-                continue
-
-            # 查询超过保留期的 superseded chunks
-            # 注意：ChromaDB 的 where 过滤可能不支持 $lt 操作符
-            # 所以我们先获取所有 superseded chunks，然后在 Python 中过滤
-            result = collection.get(
-                where={"status": "superseded"}
-            )
-
-            if not result['ids']:
-                continue
-
-            # 在 Python 中过滤超过保留期的 chunks
-            ids_to_delete = []
-            for i, meta in enumerate(result['metadatas']):
-                superseded_time = meta.get('superseded_time', '')
-                if superseded_time and superseded_time < cutoff_date:
-                    ids_to_delete.append(result['ids'][i])
-
-            if ids_to_delete:
-                # 删除这些 chunks
-                collection.delete(ids=ids_to_delete)
-                total_cleaned += len(ids_to_delete)
-                logger.info(f"清理 {kb_name}: {len(ids_to_delete)} chunks")
-
-                # 重建 BM25 索引
-                kb_manager.rebuild_bm25_index(kb_name)
-
-        except Exception as e:
-            logger.error(f"清理 {kb_name} 失败: {e}")
-            continue
-
-    logger.info(f"清理完成，共删除 {total_cleaned} 个 superseded chunks")
-    return total_cleaned
-

 def cleanup_deprecated_versions(days_to_keep: int = 30) -> int:
    """
--- a/knowledge/collection.py
+++ b/knowledge/collection.py
@@ -299,6 +299,29 @@ class CollectionMixin:
            except Exception as e:
                logger.warning(f"清理哈希记录失败: {e}")

+            # 清理该向量库的 SQLite 版本记录和变更日志
+            try:
+                from data.db import get_connection
+                with get_connection("knowledge") as conn:
+                    cursor1 = conn.execute(
+                        "DELETE FROM document_versions WHERE collection = ?",
+                        (kb_name,)
+                    )
+                    cursor2 = conn.execute(
+                        "DELETE FROM version_change_logs WHERE collection = ?",
+                        (kb_name,)
+                    )
+                    conn.commit()
+                    ver_cleaned = cursor1.rowcount
+                    log_cleaned = cursor2.rowcount
+                    if ver_cleaned > 0 or log_cleaned > 0:
+                        logger.info(
+                            f"清理向量库版本记录: {kb_name}, "
+                            f"版本 {ver_cleaned} 条, 日志 {log_cleaned} 条"
+                        )
+            except Exception as e:
+                logger.warning(f"清理版本记录失败: {e}")
+
            if kb_name in self._metadata.get("collections", {}):
                del self._metadata["collections"][kb_name]
            self._save_metadata()
--- a/knowledge/document.py
+++ b/knowledge/document.py
@@ -43,7 +43,7 @@ class DocumentMixin:
        ]

    def delete_document(self, kb_name: str, filename: str) -> int:
-        """从向量库删除文档"""
+        """从向量库删除文档，并清理 SQLite 版本记录"""
        collection = self.get_collection(kb_name)
        if not collection:
            return 0
@@ -56,6 +56,22 @@ class DocumentMixin:
        collection.delete(ids=result['ids'])
        deleted = len(result['ids'])

+        # 清理 SQLite 版本记录和变更日志
+        try:
+            from data.db import get_connection
+            with get_connection("knowledge") as conn:
+                conn.execute(
+                    "DELETE FROM document_versions WHERE collection = ? AND document_id = ?",
+                    (kb_name, filename)
+                )
+                conn.execute(
+                    "DELETE FROM version_change_logs WHERE collection = ? AND document_id = ?",
+                    (kb_name, filename)
+                )
+                conn.commit()
+        except Exception as e:
+            logger.warning(f"清理版本记录失败: {e}")
+
        logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
        return deleted

@@ -66,7 +82,7 @@ class DocumentMixin:
        reason: str = "制度废止",
        deprecated_by: str = ""
    ) -> Dict:
-        """软删除文档 - 将chunks状态标记为deprecated"""
+        """软删除文档 - 将chunks状态标记为deprecated，并同步 SQLite 版本记录"""
        collection = self.get_collection(kb_name)
        if not collection:
            return {"success": False, "error": "向量库不存在"}
@@ -97,6 +113,30 @@ class DocumentMixin:

        logger.info(f"软删除文档: {kb_name}/{filename}, chunks: {len(result['ids'])}, 原因: {reason}")

+        # 同步 SQLite 版本记录
+        try:
+            from knowledge.document_versions import get_version_query
+            from data.db import get_connection
+            vq = get_version_query()
+            active = vq.get_active_version(kb_name, filename)
+            if active:
+                with get_connection("knowledge") as conn:
+                    conn.execute("""
+                        UPDATE document_versions
+                        SET status='deprecated', deprecated_date=?, deprecated_reason=?
+                        WHERE collection=? AND document_id=? AND version=?
+                    """, (deprecated_date, reason, kb_name, filename, active.version))
+                    conn.commit()
+                vq.log_version_change(
+                    kb_name, filename,
+                    change_type="deprecate",
+                    old_version=active.version,
+                    old_status="active", new_status="deprecated",
+                    reason=reason, changed_by=deprecated_by
+                )
+        except Exception as e:
+            logger.warning(f"同步版本记录失败: {e}")
+
        return {
            "success": True,
            "deprecated_chunks": len(result['ids']),
@@ -106,7 +146,7 @@ class DocumentMixin:
        }

    def restore_document(self, kb_name: str, filename: str) -> Dict:
-        """恢复已废止的文档"""
+        """恢复已废止的文档，并同步 SQLite 版本记录"""
        collection = self.get_collection(kb_name)
        if not collection:
            return {"success": False, "error": "向量库不存在"}
@@ -142,6 +182,32 @@ class DocumentMixin:

        logger.info(f"恢复文档: {kb_name}/{filename}, chunks: {len(result['ids'])}")

+        # 同步 SQLite 版本记录
+        try:
+            from knowledge.document_versions import get_version_query
+            from data.db import get_connection
+            vq = get_version_query()
+            history = vq.get_document_history(kb_name, filename)
+            deprecated_ver = next(
+                (v for v in history if v.status.value == 'deprecated'), None
+            )
+            if deprecated_ver:
+                with get_connection("knowledge") as conn:
+                    conn.execute("""
+                        UPDATE document_versions
+                        SET status='active', deprecated_date=NULL, deprecated_reason=NULL
+                        WHERE collection=? AND document_id=? AND version=?
+                    """, (kb_name, filename, deprecated_ver.version))
+                    conn.commit()
+                vq.log_version_change(
+                    kb_name, filename,
+                    change_type="restore",
+                    old_version=deprecated_ver.version,
+                    old_status="deprecated", new_status="active"
+                )
+        except Exception as e:
+            logger.warning(f"同步版本记录失败: {e}")
+
        return {
            "success": True,
            "restored_chunks": len(result['ids']),
--- a/knowledge/manager.py
+++ b/knowledge/manager.py
@@ -215,6 +215,38 @@ class KnowledgeBaseManager(
        # 合并跨页表格
        chunks = self._merge_cross_page_tables(chunks)

+        filename = Path(filepath).name
+
+        # 入库前清理同名旧切片，防止重复上传导致新旧切片共存
+        existing = collection.get(where={"source": filename})
+        if existing and existing['ids']:
+            old_count = len(existing['ids'])
+            collection.delete(ids=existing['ids'])
+            logger.info(f"替换模式: 清理旧切片 {filename} -> {kb_name}, 共 {old_count} 个")
+
+            # 清理关联的 DocStore 文件
+            try:
+                docstore_dir = Path('.data/docstore')
+                if docstore_dir.exists():
+                    for ds_file in docstore_dir.glob(f'{kb_name}_{filename}_*.json'):
+                        ds_file.unlink()
+                        logger.debug(f"清理 DocStore: {ds_file.name}")
+            except Exception as e:
+                logger.warning(f"清理 DocStore 失败: {e}")
+
+            # 重建 BM25 索引（移除旧条目）
+            self._bm25_indexes.pop(kb_name, None)
+            bm25 = self.get_bm25_index(kb_name)
+            if bm25:
+                remaining = collection.get(include=["documents", "metadatas"])
+                if remaining['ids']:
+                    bm25.add_documents(
+                        remaining['ids'],
+                        remaining['documents'] or [],
+                        remaining['metadatas'] or []
+                    )
+                    self.save_bm25_index(kb_name)
+
        # 准备向量模型
        if embedding_model is None:
            from core.engine import get_engine
@@ -229,7 +261,6 @@ class KnowledgeBaseManager(
        metadatas = []
        embeddings = []

-        filename = Path(filepath).name
        doc_type = _get_doc_type(filename)

        for i, chunk in enumerate(chunks):
@@ -622,50 +653,73 @@ class KnowledgeBaseManager(
    def mark_document_as_superseded(
        self,
        kb_name: str,
-        old_filename: str,
-        new_filename: str,
+        filename: str,
+        new_version: str = "",
        reason: str = "版本更新"
    ) -> Dict:
-        """标记文档为已替代版本"""
-        collection = self.get_collection(kb_name)
-        if not collection:
-            return {"success": False, "error": "向量库不存在"}
+        """
+        标记文档旧版本为已替代（仅更新 SQLite 版本记录）

-        result = collection.get(where={"source": old_filename})
+        ChromaDB 中的旧切片由 Phase 3 去重逻辑自动清理，
+        此方法只负责在 document_versions 表中将 active 版本改为 superseded。

-        if not result['ids']:
-            return {"success": False, "error": "旧文档不存在"}
+        Args:
+            kb_name: 向量库名称
+            filename: 文件名
+            new_version: 新版本号（如 "v2"），用于日志记录
+            reason: 替代原因

+        Returns:
+            {"success": True, "superseded_version": "v1"}
+        """
        from datetime import datetime
-        superseded_date = datetime.now().isoformat()
+        try:
+            from knowledge.document_versions import get_version_query
+            from data.db import get_connection

-        updated_metadatas = [
-            {
-                **m,
-                "status": "superseded",
-                "superseded_by": new_filename,
-                "superseded_date": superseded_date,
-                "superseded_reason": reason
+            vq = get_version_query()
+            active = vq.get_active_version(kb_name, filename)
+            if not active:
+                return {"success": True, "superseded_version": None,
+                        "message": "无 active 版本需要标记"}
+
+            superseded_date = datetime.now().isoformat()
+            with get_connection("knowledge") as conn:
+                conn.execute("""
+                    UPDATE document_versions
+                    SET status = 'superseded',
+                        deprecated_date = ?,
+                        deprecated_reason = ?
+                    WHERE collection = ? AND document_id = ? AND version = ?
+                """, (superseded_date, reason,
+                      kb_name, filename, active.version))
+                conn.commit()
+
+            # 记录变更日志
+            vq.log_version_change(
+                kb_name, filename,
+                change_type="supersede",
+                old_version=active.version,
+                new_version=new_version,
+                old_status="active",
+                new_status="superseded",
+                reason=reason
+            )
+
+            logger.info(
+                f"标记版本替代: {kb_name}/{filename} "
+                f"{active.version} -> {new_version or '(待创建)'}"
+            )
+
+            return {
+                "success": True,
+                "superseded_version": active.version,
+                "new_version": new_version,
+                "collection": kb_name
            }
-            for m in result['metadatas']
-        ]
-
-        collection.update(
-            ids=result['ids'],
-            metadatas=updated_metadatas
-        )
-
-        self.rebuild_bm25_index(kb_name)
-
-        logger.info(f"标记文档替代: {old_filename} -> {new_filename}")
-
-        return {
-            "success": True,
-            "superseded_chunks": len(result['ids']),
-            "old_document": old_filename,
-            "new_document": new_filename,
-            "collection": kb_name
-        }
+        except Exception as e:
+            logger.warning(f"标记替代版本失败: {e}")
+            return {"success": False, "error": str(e)}

    # ==================== 辅助检索方法 ====================

--- a/knowledge/processing.py
+++ b/knowledge/processing.py
@@ -342,7 +342,10 @@ class ProcessingMixin:
                "meta": metadata
            }

-            doc_path = docstore_dir / f"{doc_id}.json"
+            # 使用 collection/doc_id 组合路径，防止跨库同名文件覆盖
+            coll = metadata.get('collection', '')
+            safe_id = f"{coll}_{doc_id}" if coll else doc_id
+            doc_path = docstore_dir / f"{safe_id}.json"
            with open(doc_path, 'w', encoding='utf-8') as f:
                json.dump(record, f, ensure_ascii=False, indent=2)

@@ -371,7 +374,10 @@ class ProcessingMixin:
                "meta": metadata
            }

-            doc_path = docstore_dir / f"{doc_id}.json"
+            # 使用 collection/doc_id 组合路径，防止跨库同名文件覆盖
+            coll = metadata.get('collection', '')
+            safe_id = f"{coll}_{doc_id}" if coll else doc_id
+            doc_path = docstore_dir / f"{safe_id}.json"
            with open(doc_path, 'w', encoding='utf-8') as f:
                json.dump(record, f, ensure_ascii=False, indent=2)

--- a/knowledge/search.py
+++ b/knowledge/search.py
@@ -284,11 +284,13 @@ class SearchMixin:

        all_items.sort(key=lambda x: x['score'], reverse=True)

+        # 使用 (collection, id) 复合键去重，防止跨库同名文件的结果被吞
        seen = set()
        unique_items = []
        for item in all_items:
-            if item['id'] not in seen:
-                seen.add(item['id'])
+            composite_key = (item['collection'], item['id'])
+            if composite_key not in seen:
+                seen.add(composite_key)
                unique_items.append(item)

        unique_items = unique_items[:top_k]
--- a/knowledge/sync.py
+++ b/knowledge/sync.py