fix(boundary): 修复多库边界问题、版本管理及删除清理

多库检索与存储修复: - RRF 融合去重改用 (collection, chunk_id) 复合键，修复同名文件结果被吞 - DocStore 存储路径加 collection 前缀，修复跨库同名切片数据覆盖 - search_multiple 去重改用复合键 - chunk_id 解析改用 rsplit 兼容下划线文件名上传与版本管理修复: - 同名文件上传改为覆盖模式，自动清理旧切片 - 修复首次上传不创建版本记录 - 修复覆盖上传版本号回退到 v1 - sync ADDED 分支改用动态版本号生成 - _generate_version_id 改为基于全部版本递增 - 废止/恢复操作同步 SQLite 版本记录 - mark_document_as_superseded 改为仅更新 SQLite 删除清理修复: - 删除文档时同步清理 SQLite 版本记录和变更日志 - 删除向量库时同步清理该库所有版本记录 - cleanup 改为清理 SQLite 记录而非 ChromaDB 测试: - test_version_management.py: 27 条版本管理单元测试 - test_edge_cases.py: 28 条边界用例测试 - test_upload_dedup.py: 5 条上传去重测试 - e2e_risk_test.py: 27 条端到端风险测试文档: - 新增风险边界问题修复注意事项.md（面向后端的对接文档） - 新增向量库边界风险分析.md - 更新多篇现有文档
2026-06-04 23:58:44 +08:00
parent a1a0814633
commit cb75b9b274
50 changed files with 6385 additions and 6248 deletions
--- a/api/chat_routes.py
+++ b/api/chat_routes.py
@@ -415,12 +415,16 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
    if not contexts:
        return {"answer_with_refs": answer, "citations": []}

-    # 按 chunk_id 组织 contexts
+    # 按 (collection, chunk_id) 复合键组织 contexts，防止跨库同名文件覆盖
    ctx_by_chunk = {}
    for ctx in contexts:
        meta = ctx.get('meta', {})
        chunk_id = meta.get('chunk_id') or f"{meta.get('source')}_{meta.get('chunk_index', 0)}"
-        ctx_by_chunk[chunk_id] = ctx
+        coll = meta.get('_collection') or meta.get('collection') or ''
+        composite_key = f"{coll}/{chunk_id}" if coll else chunk_id
+        # 保存原始 chunk_id，用于对外输出（ref tag / citation）
+        ctx['_raw_chunk_id'] = chunk_id
+        ctx_by_chunk[composite_key] = ctx

    # jieba 分词函数（fallback 到字符级）
    try:
@@ -485,20 +489,27 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
                if cid not in cited_set:
                    cited_set.add(cid)
                    cited_chunks_ordered.append(cid)
-            # 在段落末尾插入引用标记（多个引用连续排列）
-            ref_tags = "".join(f"[ref:{cid}]" for cid in selected_ids)
+            # 在段落末尾插入引用标记（使用原始 chunk_id，不暴露复合键）
+            ref_tags = "".join(
+                f"[ref:{ctx_by_chunk[cid].get('_raw_chunk_id', cid)}]"
+                for cid in selected_ids
+            )
            result_parts.append(f"{para}{ref_tags}{sep}")
        else:
            result_parts.append(para + sep)

-    # 构建引用列表（按出现顺序）
+    # 构建引用列表（按出现顺序），使用原始 chunk_id 构建 citation
    citations = []
-    for chunk_id in cited_chunks_ordered:
-        ctx = ctx_by_chunk.get(chunk_id)
+    for composite_key in cited_chunks_ordered:
+        ctx = ctx_by_chunk.get(composite_key)
        if ctx:
            meta = ctx.get('meta', {})
            full_content = ctx.get('doc', '')
            citation = _build_citation(meta, full_content)
+            # 确保 citation 中的 chunk_id 使用原始值（不含 collection 前缀）
+            raw_id = ctx.get('_raw_chunk_id')
+            if raw_id:
+                citation['chunk_id'] = raw_id
            citations.append(citation)

    return {
@@ -587,7 +598,7 @@ def _build_citation(meta: Dict, full_content: str = '') -> Dict[str, Any]:
        "chunk_id": chunk_id_raw,
        "chunk_index": chunk_index,  # 全局切片序号，用于精准定位文档位置
        "source": meta.get('source', ''),
-        "collection": meta.get('_collection', ''),  # 所属向量库，用于前端文档预览跳转
+        "collection": meta.get('_collection') or meta.get('collection', ''),  # 所属向量库，用于前端文档预览跳转
        "doc_type": meta.get('doc_type', 'other'),
        "section": _clean_section(meta.get('section', '')),
        "preview": meta.get('preview', ''),
@@ -1528,12 +1539,8 @@ def rag():
                    meta['_retrieval_rank'] = rank
                    # 确保 _collection 字段存在（单知识库路径下 ChromaDB 原生不返回此字段）
                    if not meta.get('_collection'):
-                        if collections and len(collections) == 1:
-                            meta['_collection'] = collections[0]
-                        elif collections:
-                            meta['_collection'] = collections[0]  # 多知识库时回退到第一个
-                        else:
-                            meta['_collection'] = 'public_kb'
+                        # 优先使用入库时写入的 collection 字段
+                        meta['_collection'] = meta.get('collection') or (collections[0] if collections else 'public_kb')
                    source_name = meta.get('source', '未知')
                    if source_name not in seen_sources or score > seen_sources[source_name]['score']:
                        doc_type = meta.get('doc_type', 'other')
--- a/api/document_routes.py
+++ b/api/document_routes.py
@@ -243,14 +243,75 @@ def upload_document() -> Tuple[Any, int]:
        filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"

    filepath = os.path.join(target_dir, filename)
-    if os.path.exists(filepath):
-        timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
-        name, ext_part = os.path.splitext(filename)
-        filename = f"{name}{timestamp}{ext_part}"
-        filepath = os.path.join(target_dir, filename)

+    # 同名文件处理：清理旧切片 + 覆盖旧文件（替代原来的时间戳重命名策略）
+    replaced = False
+    if os.path.exists(filepath):
+        replaced = True
+        # 1. 清理旧切片（ChromaDB + BM25 + DocStore）
+        kb_manager = _get_kb_manager()
+        if kb_manager:
+            old_chunks = kb_manager.get_document_chunks(collection, filename)
+            if old_chunks:
+                old_ids = [c['id'] for c in old_chunks]
+                coll_obj = kb_manager.get_collection(collection)
+                if coll_obj:
+                    coll_obj.delete(ids=old_ids)
+                    logger.info(f"替换上传: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
+
+                    # 清理关联的 DocStore 文件
+                    try:
+                        from pathlib import Path as _Path
+                        docstore_dir = _Path('.data/docstore')
+                        if docstore_dir.exists():
+                            for ds_file in docstore_dir.glob(f'{collection}_{filename}_*.json'):
+                                ds_file.unlink()
+                    except Exception as e:
+                        logger.warning(f"清理 DocStore 失败: {e}")
+
+                    # 重建 BM25 索引
+                    kb_manager.rebuild_bm25_index(collection)
+
+        # 2. 清理同步哈希记录（以便 sync 能正确检测为新文件）
+        try:
+            from knowledge.sync import SyncDatabase
+            sync_db = SyncDatabase()
+            sync_db.delete_document_hash(f"{collection}/{filename}")
+        except Exception as e:
+            logger.warning(f"清理哈希记录失败: {e}")
+
+    # 保存文件（覆盖同名旧文件）
    file.save(filepath)

+    # 版本管理：覆盖上传时标记旧版本为 superseded（新版本记录由 sync 服务统一创建）
+    if replaced:
+        try:
+            from knowledge.document_versions import get_version_query
+            from data.db import get_connection
+            vq = get_version_query()
+            active = vq.get_active_version(collection, filename)
+
+            if active:
+                with get_connection("knowledge") as conn:
+                    conn.execute("""
+                        UPDATE document_versions
+                        SET status='superseded', deprecated_date=?,
+                            deprecated_reason='重新上传覆盖'
+                        WHERE collection=? AND document_id=? AND version=?
+                    """, (datetime.now().isoformat(), collection,
+                          filename, active.version))
+                    conn.commit()
+                vq.log_version_change(
+                    collection, filename,
+                    change_type="reupload",
+                    old_version=active.version,
+                    old_status="active", new_status="superseded",
+                    reason="重新上传覆盖",
+                    changed_by=request.current_user.get('user_id', '')
+                )
+        except Exception as e:
+            logger.warning(f"标记旧版本失败: {e}")
+
    # 6. 触发向量化
    sync_status = "已保存，等待手动同步"
    sync_service = _get_sync_service()
@@ -276,7 +337,8 @@ def upload_document() -> Tuple[Any, int]:
                "filename": filename,
                "collection": collection,
                "path": f"{target_subdir}/{filename}",
-                "size": file_size
+                "size": file_size,
+                "replaced": replaced
            },
            "sync_status": sync_status
        },
@@ -350,19 +412,56 @@ def batch_upload_documents() -> Tuple[Any, int]:
                filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
            filepath = os.path.join(target_dir, filename)

-            # 处理重名
+            # 同名文件处理：清理旧切片 + 覆盖旧文件
+            replaced = False
            if os.path.exists(filepath):
-                timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
-                name, ext_part = os.path.splitext(filename)
-                filename = f"{name}{timestamp}{ext_part}"
-                filepath = os.path.join(target_dir, filename)
+                replaced = True
+                kb_mgr = _get_kb_manager()
+                if kb_mgr:
+                    old_chunks = kb_mgr.get_document_chunks(collection, filename)
+                    if old_chunks:
+                        old_ids = [c['id'] for c in old_chunks]
+                        coll_obj = kb_mgr.get_collection(collection)
+                        if coll_obj:
+                            coll_obj.delete(ids=old_ids)
+                            logger.info(f"批量替换: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
+                    kb_mgr.rebuild_bm25_index(collection)
+
+                try:
+                    from knowledge.sync import SyncDatabase
+                    sync_db = SyncDatabase()
+                    sync_db.delete_document_hash(f"{collection}/{filename}")
+                except Exception as e:
+                    logger.warning(f"清理哈希记录失败: {e}")

            file.save(filepath)

+            # 版本管理：覆盖上传时标记旧版本为 superseded（新版本记录由 sync 服务统一创建）
+            if replaced:
+                try:
+                    from knowledge.document_versions import get_version_query
+                    from data.db import get_connection
+                    vq = get_version_query()
+                    active = vq.get_active_version(collection, filename)
+
+                    if active:
+                        with get_connection("knowledge") as conn:
+                            conn.execute("""
+                                UPDATE document_versions
+                                SET status='superseded', deprecated_date=?,
+                                    deprecated_reason='批量上传覆盖'
+                                WHERE collection=? AND document_id=? AND version=?
+                            """, (datetime.now().isoformat(), collection,
+                                  filename, active.version))
+                            conn.commit()
+                except Exception as e:
+                    logger.warning(f"批量上传标记旧版本失败: {e}")
+
            results.append({
                "filename": filename,
                "status": "success",
-                "path": f"{target_subdir}/{filename}"
+                "path": f"{target_subdir}/{filename}",
+                "replaced": replaced
            })
        except Exception as e:
            results.append({