jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(boundary): 修复多库边界问题、版本管理及删除清理

Browse Source 
多库检索与存储修复:
- RRF 融合去重改用 (collection, chunk_id) 复合键，修复同名文件结果被吞
- DocStore 存储路径加 collection 前缀，修复跨库同名切片数据覆盖
- search_multiple 去重改用复合键
- chunk_id 解析改用 rsplit 兼容下划线文件名

上传与版本管理修复:
- 同名文件上传改为覆盖模式，自动清理旧切片
- 修复首次上传不创建版本记录
- 修复覆盖上传版本号回退到 v1
- sync ADDED 分支改用动态版本号生成
- _generate_version_id 改为基于全部版本递增
- 废止/恢复操作同步 SQLite 版本记录
- mark_document_as_superseded 改为仅更新 SQLite

删除清理修复:
- 删除文档时同步清理 SQLite 版本记录和变更日志
- 删除向量库时同步清理该库所有版本记录
- cleanup 改为清理 SQLite 记录而非 ChromaDB

测试:
- test_version_management.py: 27 条版本管理单元测试
- test_edge_cases.py: 28 条边界用例测试
- test_upload_dedup.py: 5 条上传去重测试
- e2e_risk_test.py: 27 条端到端风险测试

文档:
- 新增风险边界问题修复注意事项.md（面向后端的对接文档）
- 新增向量库边界风险分析.md
- 更新多篇现有文档
This commit is contained in:
lacerate551
2026-06-04 23:58:44 +08:00
parent a1a0814633
commit cb75b9b274
50 changed files with 6385 additions and 6248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

419
tests/e2e_risk_test.py Normal file
View File

@@ -0,0 +1,419 @@
# -*- coding: utf-8 -*-
"""
端到端风险测试脚本 - 直接调用运行中的服务 API
测试场景：
1. 上传同名文件覆盖 → 版本记录是否正确（P0 修复验证）
2. 废止文档 → 切片状态是否正确标记（Part 4 修复验证）
3. 废止后检索 → 废止文档不应出现在结果中（过滤逻辑验证）
4. 恢复文档 → 切片状态应恢复为 active
5. SQLite/ChromaDB 状态一致性
6. 边缘操作：重复废止、恢复非废止文档等
"""
import requests
import json
import time
import os
import sys
import tempfile
API = "http://localhost:5001"
KB = "test1" # 使用隔离的测试知识库
passed = 0
failed = 0
def check(name, condition, detail=""):
global passed, failed
if condition:
print(f" [PASS] {name}")
passed += 1
else:
print(f" [FAIL] {name} {detail}")
failed += 1
def upload_file(collection, filename, content):
"""上传文件到指定知识库"""
# 创建临时文件
tmpdir = tempfile.mkdtemp()
filepath = os.path.join(tmpdir, filename)
with open(filepath, 'w', encoding='utf-8') as f:
f.write(content)
with open(filepath, 'rb') as f:
resp = requests.post(
f"{API}/documents/upload",
files={"file": (filename, f, "text/plain")},
data={"collection": collection}
)
# 清理临时文件
os.remove(filepath)
os.rmdir(tmpdir)
return resp.json()
def get_doc_chunks(collection, filename):
"""获取文档的切片信息"""
resp = requests.get(
f"{API}/documents/{collection}/{filename}/chunks"
)
if resp.status_code == 200:
return resp.json()
return {"error": resp.text, "chunks": []}
def get_doc_status(collection, filename):
"""获取文档状态"""
resp = requests.get(
f"{API}/documents/{collection}/{filename}/status"
)
if resp.status_code == 200:
return resp.json()
return {"error": resp.text}
def get_version_history(collection, filename):
"""获取版本历史"""
resp = requests.get(
f"{API}/collections/{collection}/documents/{filename}/versions"
)
if resp.status_code == 200:
return resp.json()
return {"error": resp.text, "versions": []}
def deprecate_doc(collection, filename, reason="测试废止"):
"""废止文档"""
resp = requests.post(
f"{API}/collections/{collection}/documents/{filename}/deprecate",
json={"reason": reason}
)
return resp.json()
def restore_doc(collection, filename):
"""恢复文档"""
resp = requests.post(
f"{API}/collections/{collection}/documents/{filename}/restore"
)
return resp.json()
def delete_doc(collection, filename):
"""删除文档"""
resp = requests.delete(
f"{API}/documents/{collection}/{filename}"
)
return resp.json()
def rag_query(collection, query):
"""发送 RAG 查询"""
resp = requests.post(
f"{API}/rag",
json={
"question": query,
"collection": collection,
"stream": False
},
timeout=30
)
if resp.status_code == 200:
return resp.json()
return {"error": resp.text}
# ======================================================================
# 清理：确保 test1 知识库是干净的
# ======================================================================
print("\n=== 准备：清理 test1 知识库 ===")
list_resp = requests.get(f"{API}/documents/list?collection={KB}").json()
for doc in list_resp.get("documents", []):
src = doc.get("source", "")
if src:
delete_doc(KB, src)
print(f" 清理旧文档: {src}")
time.sleep(1)
print(" 知识库已清理")
# ======================================================================
# 风险场景 1：上传同名文件覆盖 → 版本记录
# ======================================================================
print("\n=== 风险场景 1：上传覆盖 → SQLite 版本记录 ===")
# 1a. 上传 v1
v1_content = "这是版本测试文档的第一版内容。\n包含一些独特的v1信息用于后续检索验证。\nAlpha Bravo Charlie Delta."
r1 = upload_file(KB, "version_test.txt", v1_content)
check("v1 上传成功", r1.get("success") is True or "chunks" in str(r1),
f"resp: {json.dumps(r1, ensure_ascii=False)[:200]}")
time.sleep(1)
# 1b. 检查 v1 的 chunks
chunks_v1 = get_doc_chunks(KB, "version_test.txt")
v1_chunks_list = chunks_v1.get("chunks", [])
check("v1 有切片",
len(v1_chunks_list) > 0,
f"chunks count: {len(v1_chunks_list)}")
if v1_chunks_list:
first_meta = v1_chunks_list[0].get("metadata", {})
check("v1 切片 status=active",
first_meta.get("status") == "active",
f"status: {first_meta.get('status')}")
# 1c. 上传 v2（同名覆盖）
v2_content = "这是版本测试文档的第二版内容。\nv2版本包含了全新的信息。\nEcho Foxtrot Golf Hotel."
r2 = upload_file(KB, "version_test.txt", v2_content)
check("v2 覆盖上传成功",
r2.get("success") is True or "chunks" in str(r2),
f"resp: {json.dumps(r2, ensure_ascii=False)[:200]}")
check("v2 标记为 replaced",
r2.get("data", {}).get("file", {}).get("replaced") is True,
f"replaced: {r2.get('data', {}).get('file', {}).get('replaced')}")
time.sleep(1)
# 1d. 检查 v2 的 chunks（应该是新版本内容）
chunks_v2 = get_doc_chunks(KB, "version_test.txt")
v2_chunks_list = chunks_v2.get("chunks", [])
check("v2 有切片",
len(v2_chunks_list) > 0,
f"chunks count: {len(v2_chunks_list)}")
if v2_chunks_list:
first_meta = v2_chunks_list[0].get("metadata", {})
check("v2 切片 status=active",
first_meta.get("status") == "active",
f"status: {first_meta.get('status')}")
# 1e. 检查版本历史（SQLite）
versions = get_version_history(KB, "version_test.txt")
ver_list = versions.get("versions", [])
check("版本历史有记录",
len(ver_list) > 0,
f"versions: {json.dumps(ver_list, ensure_ascii=False)[:300]}")
if len(ver_list) >= 2:
# 应该有 v1(superseded) 和 v2(active)
statuses = [v.get("status") for v in ver_list]
check("版本历史包含 superseded 和 active 状态",
"superseded" in str(statuses) and "active" in str(statuses),
f"statuses: {statuses}")
elif len(ver_list) == 1:
check("至少有一条 active 版本记录",
ver_list[0].get("status") in ("active", "superseded"),
f"version: {ver_list[0]}")
# ======================================================================
# 风险场景 2：废止文档 → 切片状态标记
# ======================================================================
print("\n=== 风险场景 2：废止文档 → ChromaDB + SQLite 状态同步 ===")
# 2a. 上传一个专门用于废止测试的文档
deprecate_content = "这份文档将被废止。\n包含独特的废止测试关键词 XYZ123ABC。\nIndigo Juliet Kilo Lima."
r3 = upload_file(KB, "deprecate_test.txt", deprecate_content)
check("废止测试文档上传成功",
r3.get("success") is True or "chunks" in str(r3),
f"resp: {json.dumps(r3, ensure_ascii=False)[:200]}")
time.sleep(1)
# 2b. 确认上传后状态为 active
chunks_before = get_doc_chunks(KB, "deprecate_test.txt")
if chunks_before.get("chunks"):
check("上传后切片状态为 active",
all(c.get("metadata", {}).get("status") == "active"
for c in chunks_before["chunks"]),
f"statuses: {[c.get('metadata', {}).get('status') for c in chunks_before['chunks']]}")
# 2c. 执行废止
dep_result = deprecate_doc(KB, "deprecate_test.txt", reason="测试废止操作")
check("废止操作返回 success",
dep_result.get("success") is True,
f"resp: {json.dumps(dep_result, ensure_ascii=False)[:200]}")
check("废止标记了切片",
dep_result.get("deprecated_chunks", 0) > 0,
f"deprecated_chunks: {dep_result.get('deprecated_chunks')}")
# 2d. 验证 ChromaDB 中的切片状态
time.sleep(0.5)
chunks_after_dep = get_doc_chunks(KB, "deprecate_test.txt")
if chunks_after_dep.get("chunks"):
dep_statuses = [c.get("metadata", {}).get("status", "") for c in chunks_after_dep["chunks"]]
check("ChromaDB 切片状态已改为 deprecated",
all(s == "deprecated" for s in dep_statuses),
f"statuses: {dep_statuses}")
# 2e. 验证 SQLite 版本记录也同步了
dep_versions = get_version_history(KB, "deprecate_test.txt")
dep_ver_list = dep_versions.get("versions", [])
if dep_ver_list:
has_deprecated = any(
v.get("status") in ("deprecated",) or
str(v.get("status", "")).lower() == "deprecated"
for v in dep_ver_list
)
check("SQLite 版本记录中有 deprecated 状态",
has_deprecated,
f"versions: {json.dumps(dep_ver_list, ensure_ascii=False)[:300]}")
else:
check("SQLite 版本记录存在", False, "版本历史为空")
# ======================================================================
# 风险场景 3：废止后检索 → 不应出现废止文档内容
# ======================================================================
print("\n=== 风险场景 3：废止后检索过滤 ===")
# 用废止文档中的独特关键词检索
search_result = rag_query(KB, "XYZ123ABC 废止测试关键词")
if "error" not in search_result:
answer = search_result.get("answer", "")
sources = search_result.get("sources", [])
source_files = [s.get("source", s.get("file", "")) for s in sources] if sources else []
check("检索结果不包含废止文档",
"deprecate_test.txt" not in source_files,
f"sources: {source_files}")
# 也检查 citations
citations = search_result.get("citations", [])
cite_sources = [c.get("source", "") for c in citations] if citations else []
check("citations 不包含废止文档",
"deprecate_test.txt" not in cite_sources,
f"cite_sources: {cite_sources}")
else:
print(f" [SKIP] RAG 查询失败（可能 LLM 不可用）: {search_result.get('error', '')[:100]}")
# 备选方案：直接检查 chunks 的 status 字段
dep_check = get_doc_chunks(KB, "deprecate_test.txt")
if dep_check.get("chunks"):
all_dep = all(
c.get("status") == "deprecated" or c.get("metadata", {}).get("status") == "deprecated"
for c in dep_check["chunks"]
)
check("（备选）废止文档所有切片 status=deprecated",
all_dep,
f"statuses: {[c.get('status', c.get('metadata', {}).get('status')) for c in dep_check['chunks']]}")
# ======================================================================
# 风险场景 4：恢复文档 → 切片状态恢复
# ======================================================================
print("\n=== 风险场景 4：恢复已废止文档 ===")
restore_result = restore_doc(KB, "deprecate_test.txt")
check("恢复操作返回 success",
restore_result.get("success") is True,
f"resp: {json.dumps(restore_result, ensure_ascii=False)[:200]}")
check("恢复了切片",
restore_result.get("restored_chunks", 0) > 0,
f"restored_chunks: {restore_result.get('restored_chunks')}")
# 验证 ChromaDB 切片恢复
time.sleep(0.5)
chunks_after_restore = get_doc_chunks(KB, "deprecate_test.txt")
if chunks_after_restore.get("chunks"):
restored_statuses = [c.get("metadata", {}).get("status", "") for c in chunks_after_restore["chunks"]]
check("ChromaDB 切片状态恢复为 active",
all(s == "active" for s in restored_statuses),
f"statuses: {restored_statuses}")
# 验证 SQLite 版本记录
rest_versions = get_version_history(KB, "deprecate_test.txt")
rest_ver_list = rest_versions.get("versions", [])
if rest_ver_list:
has_active = any(
str(v.get("status", "")).lower() == "active"
for v in rest_ver_list
)
check("SQLite 版本记录恢复为 active",
has_active,
f"versions: {json.dumps(rest_ver_list, ensure_ascii=False)[:300]}")
# ======================================================================
# 风险场景 5：SQLite 与 ChromaDB 状态一致性
# ======================================================================
print("\n=== 风险场景 5：SQLite/ChromaDB 状态一致性 ===")
# 对 version_test.txt 做一致性检查
ver_chunks = get_doc_chunks(KB, "version_test.txt")
ver_versions = get_version_history(KB, "version_test.txt")
if ver_chunks.get("chunks") and ver_versions.get("versions"):
# ChromaDB 中所有切片应该是 active（只有当前版本在 ChromaDB 中）
chroma_statuses = set(
c.get("metadata", {}).get("status", "active")
for c in ver_chunks["chunks"]
)
check("ChromaDB 中 version_test 切片全为 active",
chroma_statuses == {"active"} or chroma_statuses == set(),
f"chroma_statuses: {chroma_statuses}")
# SQLite 中应该至少有一条 active 记录
sqlite_statuses = [v.get("status") for v in ver_versions["versions"]]
check("SQLite 中有 active 版本记录",
"active" in sqlite_statuses,
f"sqlite_statuses: {sqlite_statuses}")
else:
check("能获取到切片和版本信息", False,
f"chunks: {bool(ver_chunks.get('chunks'))}, versions: {bool(ver_versions.get('versions'))}")
# ======================================================================
# 风险场景 6：边缘操作（不应崩溃）
# ======================================================================
print("\n=== 风险场景 6：边缘操作容错 ===")
# 6a. 废止不存在的文档
dep_nonexist = deprecate_doc(KB, "nonexistent_file_xyz.txt")
check("废止不存在文档不崩溃",
"success" in dep_nonexist or "error" in dep_nonexist,
f"resp: {json.dumps(dep_nonexist, ensure_ascii=False)[:200]}")
# 6b. 恢复非废止状态的文档
# deprecate_test.txt 已经被恢复了，再恢复一次应该报错但不崩溃
restore_again = restore_doc(KB, "deprecate_test.txt")
check("重复恢复不崩溃",
"success" in restore_again or "error" in restore_again,
f"resp: {json.dumps(restore_again, ensure_ascii=False)[:200]}")
# 6c. 连续两次废止同一文档
dep1 = deprecate_doc(KB, "deprecate_test.txt", reason="第一次废止")
check("第一次废止成功",
dep1.get("success") is True,
f"resp: {json.dumps(dep1, ensure_ascii=False)[:200]}")
time.sleep(0.5)
dep2 = deprecate_doc(KB, "deprecate_test.txt", reason="第二次废止")
check("第二次废止不崩溃（已废止状态）",
"success" in dep2 or "error" in dep2,
f"resp: {json.dumps(dep2, ensure_ascii=False)[:200]}")
# 恢复（为后续清理准备）
restore_doc(KB, "deprecate_test.txt")
time.sleep(0.5)
# 6d. 上传空文件
empty_result = upload_file(KB, "empty_file.txt", "")
check("上传空文件不崩溃",
"success" in empty_result or "error" in empty_result,
f"resp: {json.dumps(empty_result, ensure_ascii=False)[:200]}")
# ======================================================================
# 清理：删除测试文档
# ======================================================================
print("\n=== 清理测试数据 ===")
for fname in ["version_test.txt", "deprecate_test.txt", "empty_file.txt"]:
r = delete_doc(KB, fname)
print(f" 删除 {fname}: {r.get('success', r.get('deleted', r.get('error', '?')))}")
# ======================================================================
# 汇总
# ======================================================================
print(f"\n{'='*60}")
print(f"风险测试完成: {passed} 通过, {failed} 失败, 共 {passed + failed} 条")
print(f"{'='*60}")
if failed > 0:
sys.exit(1)

330
tests/test_edge_cases.py Normal file
View File

@@ -0,0 +1,330 @@
# -*- coding: utf-8 -*-
"""
Phase 1 + Phase 2 边界风险修复验证测试
测试覆盖：
1. RRF 融合跨库同名文件不吞结果
2. search_multiple 跨库去重正确
3. DocStore 路径含 collection 前缀
4. citation 构建兼容 _collection 和 collection
5. _collection 回退逻辑正确
"""
import sys
import os
import json
import tempfile
import shutil
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
passed = 0
failed = 0
def check(name, condition, detail=""):
global passed, failed
if condition:
print(f" [PASS] {name}")
passed += 1
else:
print(f" [FAIL] {name} {detail}")
failed += 1
# ======================================================================
# 测试 1：RRF 融合 — 跨库同名文件不被吞
# ======================================================================
print("\n=== 测试 1：RRF 融合复合键去重 ===")
# 模拟 engine 的 reciprocal_rank_fusion 逻辑（提取纯函数测试）
def reciprocal_rank_fusion_test(results_list, weights=None, k=60):
"""复刻 engine.py 的 RRF 逻辑（含修复）"""
if not results_list:
return {'ids': [[]], 'documents': [[]], 'metadatas': [[]], 'distances': [[]]}
if weights is None:
weights = [1.0] * len(results_list)
doc_scores = {}
for results, weight in zip(results_list, weights):
if not results['documents'] or not results['documents'][0]:
continue
for rank, (doc_id, doc, meta) in enumerate(zip(
results['ids'][0], results['documents'][0], results['metadatas'][0]
)):
rrf_score = weight / (k + rank + 1)
coll = meta.get('_collection') or meta.get('collection') or ''
composite_key = f"{coll}\x00{doc_id}" if coll else doc_id
if composite_key not in doc_scores:
doc_scores[composite_key] = {'score': 0.0, 'doc': doc, 'meta': meta, 'coll': coll, 'raw_id': doc_id}
doc_scores[composite_key]['score'] += rrf_score
sorted_items = sorted(doc_scores.items(), key=lambda x: x[1]['score'], reverse=True)
out_ids = []
for item in sorted_items:
coll = item[1]['coll']
raw_id = item[1]['raw_id']
if coll and not raw_id.startswith(f"{coll}/"):
out_ids.append(f"{coll}/{raw_id}")
else:
out_ids.append(raw_id)
return {
'ids': [out_ids],
'documents': [[item[1]['doc'] for item in sorted_items]],
'metadatas': [[item[1]['meta'] for item in sorted_items]],
'distances': [[item[1]['score'] for item in sorted_items]],
}
# 场景：public_kb 和 dept_1_kb 都有 "规章制度.pdf_0"
result_public = {
'ids': [['规章制度.pdf_0', '规章制度.pdf_1']],
'documents': [['public版内容_0', 'public版内容_1']],
'metadatas': [[
{'source': '规章制度.pdf', 'collection': 'public_kb', '_collection': 'public_kb', 'chunk_index': 0},
{'source': '规章制度.pdf', 'collection': 'public_kb', '_collection': 'public_kb', 'chunk_index': 1},
]],
'distances': [[0.9, 0.8]],
}
result_dept = {
'ids': [['规章制度.pdf_0', '规章制度.pdf_1']],
'documents': [['dept版内容_0', 'dept版内容_1']],
'metadatas': [[
{'source': '规章制度.pdf', 'collection': 'dept_1_kb', '_collection': 'dept_1_kb', 'chunk_index': 0},
{'source': '规章制度.pdf', 'collection': 'dept_1_kb', '_collection': 'dept_1_kb', 'chunk_index': 1},
]],
'distances': [[0.85, 0.75]],
}
rrf_result = reciprocal_rank_fusion_test([result_public, result_dept])
rrf_ids = rrf_result['ids'][0]
check("跨库同名文件：结果数应为 4（不是 2）", len(rrf_ids) == 4, f"实际: {len(rrf_ids)}")
check("ID 包含 collection 前缀", all('/' in i for i in rrf_ids), f"IDs: {rrf_ids}")
check("public_kb 的结果存在", any('public_kb/' in i for i in rrf_ids))
check("dept_1_kb 的结果存在", any('dept_1_kb/' in i for i in rrf_ids))
# 验证文档内容保留完整（不被覆盖）
rrf_docs = rrf_result['documents'][0]
check("public版内容保留", any('public版' in d for d in rrf_docs))
check("dept版内容保留", any('dept版' in d for d in rrf_docs))
# 场景：同库向量+BM25同名chunk应合并分数
result_vec = {
'ids': [['规章制度.pdf_0']],
'documents': [['内容A']],
'metadatas': [[{'source': '规章制度.pdf', '_collection': 'public_kb', 'collection': 'public_kb'}]],
'distances': [[0.9]],
}
result_bm25 = {
'ids': [['规章制度.pdf_0']],
'documents': [['内容A']],
'metadatas': [[{'source': '规章制度.pdf', '_collection': 'public_kb', 'collection': 'public_kb'}]],
'distances': [[0.7]],
}
rrf_same = reciprocal_rank_fusion_test([result_vec, result_bm25])
check("同库向量+BM25合并：结果数应为 1", len(rrf_same['ids'][0]) == 1, f"实际: {len(rrf_same['ids'][0])}")
# chunk_index 解析测试（带前缀的 ID）
print("\n--- chunk_index 解析兼容性 ---")
for test_id, expected in [
("public_kb/规章制度.pdf_0", 0),
("dept_1_kb/规章制度.pdf_5", 5),
("规章制度.pdf_3", 3),
("test_.pdf_0", 0),
]:
chunk_id_raw = test_id
try:
chunk_index = int(str(chunk_id_raw).rsplit('_', 1)[-1])
except (ValueError, IndexError):
chunk_index = None
check(f"解析 '{test_id}' -> chunk_index={expected}", chunk_index == expected, f"实际: {chunk_index}")
# ======================================================================
# 测试 2：search_multiple 去重
# ======================================================================
print("\n=== 测试 2：search_multiple 复合键去重 ===")
# 模拟 search.py 的 _merge_multiple_results 逻辑
class FakeSearchResult:
def __init__(self, ids, documents, metadatas, distances, collection_name=""):
self.ids = ids
self.documents = documents
self.metadatas = metadatas
self.distances = distances
self.collection_name = collection_name
def merge_multiple_results_test(results, top_k=10):
"""复刻 search.py 的 _merge_multiple_results 逻辑（含修复）"""
if not results:
return []
all_items = []
for result in results:
for i, doc_id in enumerate(result.ids):
all_items.append({
'id': doc_id,
'doc': result.documents[i],
'meta': result.metadatas[i],
'score': result.distances[i],
'collection': result.collection_name
})
all_items.sort(key=lambda x: x['score'], reverse=True)
seen = set()
unique_items = []
for item in all_items:
composite_key = (item['collection'], item['id'])
if composite_key not in seen:
seen.add(composite_key)
unique_items.append(item)
return unique_items[:top_k]
result_a = FakeSearchResult(
ids=['规章制度.pdf_0', '规章制度.pdf_1'],
documents=['A内容0', 'A内容1'],
metadatas=[{'source': '规章制度.pdf'}, {'source': '规章制度.pdf'}],
distances=[0.9, 0.8],
collection_name='public_kb'
)
result_b = FakeSearchResult(
ids=['规章制度.pdf_0', '规章制度.pdf_1'],
documents=['B内容0', 'B内容1'],
metadatas=[{'source': '规章制度.pdf'}, {'source': '规章制度.pdf'}],
distances=[0.85, 0.75],
collection_name='dept_1_kb'
)
merged = merge_multiple_results_test([result_a, result_b])
check("跨库去重：结果数应为 4", len(merged) == 4, f"实际: {len(merged)}")
check("包含 public_kb 的内容", any('A内容' in m['doc'] for m in merged))
check("包含 dept_1_kb 的内容", any('B内容' in m['doc'] for m in merged))
# 同库应去重
result_a2 = FakeSearchResult(
ids=['规章制度.pdf_0'],
documents=['A内容0-vec'],
metadatas=[{'source': '规章制度.pdf'}],
distances=[0.9],
collection_name='public_kb'
)
result_a3 = FakeSearchResult(
ids=['规章制度.pdf_0'],
documents=['A内容0-bm25'],
metadatas=[{'source': '规章制度.pdf'}],
distances=[0.7],
collection_name='public_kb'
)
merged_same = merge_multiple_results_test([result_a2, result_a3])
check("同库去重：结果数应为 1", len(merged_same) == 1, f"实际: {len(merged_same)}")
# ======================================================================
# 测试 3：DocStore 路径含 collection 前缀
# ======================================================================
print("\n=== 测试 3：DocStore 路径生成 ===")
def docstore_path_test(doc_id, metadata):
"""复刻 processing.py 的路径逻辑"""
coll = metadata.get('collection', '')
safe_id = f"{coll}_{doc_id}" if coll else doc_id
return f"{safe_id}.json"
path1 = docstore_path_test("规章制度.pdf_2", {"collection": "public_kb"})
path2 = docstore_path_test("规章制度.pdf_2", {"collection": "dept_1_kb"})
check("跨库路径不同", path1 != path2, f"path1={path1}, path2={path2}")
check("public_kb 路径含前缀", path1 == "public_kb_规章制度.pdf_2.json", f"实际: {path1}")
check("dept_1_kb 路径含前缀", path2 == "dept_1_kb_规章制度.pdf_2.json", f"实际: {path2}")
# 无 collection 的兼容
path3 = docstore_path_test("test.pdf_0", {})
check("无 collection 时保持原始路径", path3 == "test.pdf_0.json", f"实际: {path3}")
# ======================================================================
# 测试 4：citation 构建 — collection 字段兼容
# ======================================================================
print("\n=== 测试 4：citation collection 字段兼容 ===")
def citation_collection_test(meta):
"""复刻 chat_routes.py 的 collection 取值逻辑"""
return meta.get('_collection') or meta.get('collection', '')
# 场景 A：只有 _collection（多库检索路径）
check("_collection 优先", citation_collection_test({'_collection': 'dept_1_kb', 'collection': 'public_kb'}) == 'dept_1_kb')
# 场景 B：只有 collection（入库时的 metadata）
check("回退到 collection", citation_collection_test({'collection': 'public_kb'}) == 'public_kb')
# 场景 C：两者都没有
check("都没有时返回空", citation_collection_test({}) == '')
# 场景 D：_collection 为空字符串
check("_collection 空字符串时回退", citation_collection_test({'_collection': '', 'collection': 'public_kb'}) == 'public_kb')
# ======================================================================
# 测试 5：_collection 回退逻辑
# ======================================================================
print("\n=== 测试 5：_collection 回退逻辑 ===")
def fallback_collection_test(meta, collections):
"""复刻 chat_routes.py 修复后的回退逻辑"""
if not meta.get('_collection'):
meta['_collection'] = meta.get('collection') or (collections[0] if collections else 'public_kb')
return meta['_collection']
# 场景：meta 有 collection 字段但无 _collection
m1 = {'collection': 'dept_1_kb'}
check("使用 meta 中的 collection", fallback_collection_test(m1, ['public_kb', 'dept_1_kb']) == 'dept_1_kb')
# 场景：meta 两者都没有
m2 = {}
check("回退到 collections[0]", fallback_collection_test(m2, ['public_kb', 'dept_1_kb']) == 'public_kb')
# 场景：已有 _collection 不覆盖
m3 = {'_collection': 'dept_1_kb', 'collection': 'public_kb'}
check("已有 _collection 不覆盖", fallback_collection_test(m3, ['public_kb']) == 'dept_1_kb')
# ======================================================================
# 测试 6：_expand_contiguous_chunks ID 匹配兼容
# ======================================================================
print("\n=== 测试 6：上下文扩展 ID 匹配兼容 ===")
# 模拟 existing_ids 和 n_id 的匹配逻辑
existing_ids = {'public_kb/规章制度.pdf_0', 'public_kb/规章制度.pdf_1', 'dept_1_kb/制度.pdf_3'}
_raw_id_set = set()
for _eid in existing_ids:
if '/' in _eid:
_raw_id_set.add(_eid.split('/', 1)[1])
else:
_raw_id_set.add(_eid)
# 邻居查询返回的是原始 ID（无前缀）
n_id_raw = "规章制度.pdf_0"
n_id_new = "规章制度.pdf_5"
is_dup_raw = n_id_raw in existing_ids or n_id_raw in _raw_id_set
is_dup_new = n_id_new in existing_ids or n_id_new in _raw_id_set
check("原始 ID 能匹配带前缀的 existing_ids", is_dup_raw == True)
check("新 ID 不误判为已存在", is_dup_new == False)
# ======================================================================
# 汇总
# ======================================================================
print(f"\n{'='*50}")
print(f"测试结果：{passed} 通过 / {failed} 失败 / {passed+failed} 总计")
if failed > 0:
print("存在失败项，请检查！")
sys.exit(1)
else:
print("全部通过！")

375
tests/test_upload_dedup.py Normal file
View File

@@ -0,0 +1,375 @@
"""
Phase 3 验证测试：重复上传旧切片残留修复
测试场景：
1. add_file_to_kb() 中的"先删后加"逻辑
2. upload_document() 中的同名文件覆盖逻辑
3. 批量上传中的同名文件处理
"""
import os
import sys
import tempfile
import shutil
from pathlib import Path
# 添加项目根目录到 Python 路径
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
def test_add_file_to_kb_dedup():
"""测试 add_file_to_kb() 中的去重逻辑"""
print("\n=== 测试 1：add_file_to_kb 去重逻辑 ===")
# 模拟 ChromaDB collection
class MockCollection:
def __init__(self):
self.data = {} # id -> {doc, meta, embedding}
def add(self, ids, documents, metadatas, embeddings):
for i, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
self.data[i] = {'doc': doc, 'meta': meta, 'embedding': emb}
def get(self, where=None, ids=None, include=None):
results = {'ids': [], 'documents': [], 'metadatas': [], 'embeddings': []}
if ids:
for id in ids:
if id in self.data:
results['ids'].append(id)
results['documents'].append(self.data[id]['doc'])
results['metadatas'].append(self.data[id]['meta'])
results['embeddings'].append(self.data[id]['embedding'])
return results
for id, item in self.data.items():
if where:
# 简单的 where 过滤
match = True
for k, v in where.items():
if item['meta'].get(k) != v:
match = False
break
if not match:
continue
results['ids'].append(id)
results['documents'].append(item['doc'])
results['metadatas'].append(item['meta'])
results['embeddings'].append(item['embedding'])
return results
def delete(self, ids):
for id in ids:
self.data.pop(id, None)
def count(self):
return len(self.data)
# 模拟知识库管理器（仅测试去重逻辑）
class MockKBManager:
def __init__(self):
self.collections = {'test_kb': MockCollection()}
self._bm25_indexes = {}
def get_collection(self, kb_name):
return self.collections.get(kb_name)
def rebuild_bm25_index(self, kb_name):
pass
def add_file_to_kb_logic(self, kb_name, filename, chunks):
"""模拟 add_file_to_kb 的核心逻辑"""
collection = self.get_collection(kb_name)
if not collection:
return 0
# 入库前清理同名旧切片（核心修复）
existing = collection.get(where={"source": filename})
if existing and existing['ids']:
old_count = len(existing['ids'])
collection.delete(ids=existing['ids'])
print(f" [清理] 删除旧切片: {filename}, 共 {old_count} 个")
# 添加新切片
ids = []
documents = []
metadatas = []
embeddings = []
for i, chunk in enumerate(chunks):
chunk_id = f"{filename}_{i}"
ids.append(chunk_id)
documents.append(chunk)
metadatas.append({
"source": filename,
"chunk_index": i,
"collection": kb_name
})
embeddings.append([0.1] * 768) # 模拟向量
collection.add(ids, documents, metadatas, embeddings)
return len(ids)
# 测试场景 1：首次添加
kb = MockKBManager()
count1 = kb.add_file_to_kb_logic('test_kb', '规章制度.pdf', ['内容1', '内容2', '内容3'])
assert count1 == 3, f"首次添加应返回 3，实际 {count1}"
assert kb.collections['test_kb'].count() == 3, f"向量库应有 3 个切片，实际 {kb.collections['test_kb'].count()}"
print(f" [PASS] 首次添加: {count1} 个切片")
# 测试场景 2：重复上传同名文件（应清理旧切片）
count2 = kb.add_file_to_kb_logic('test_kb', '规章制度.pdf', ['新内容1', '新内容2'])
assert count2 == 2, f"重复上传应返回 2，实际 {count2}"
assert kb.collections['test_kb'].count() == 2, f"向量库应有 2 个切片（旧切片已清理），实际 {kb.collections['test_kb'].count()}"
print(f" [PASS] 重复上传: {count2} 个切片（旧切片已清理）")
# 验证切片内容是新的
result = kb.collections['test_kb'].get()
assert '新内容1' in result['documents'], "切片内容应为新内容"
assert '内容1' not in result['documents'], "旧内容应已被删除"
print(f" [PASS] 切片内容已更新为新版本")
# 测试场景 3：不同名文件不互相影响
count3 = kb.add_file_to_kb_logic('test_kb', '操作手册.pdf', ['手册1'])
assert count3 == 1, f"添加新文件应返回 1，实际 {count3}"
assert kb.collections['test_kb'].count() == 3, f"向量库应有 3 个切片（2+1），实际 {kb.collections['test_kb'].count()}"
print(f" [PASS] 不同名文件独立存在: 总计 {kb.collections['test_kb'].count()} 个切片")
def test_upload_document_overwrite():
"""测试 upload_document() 中的同名文件覆盖逻辑"""
print("\n=== 测试 2：upload_document 同名文件覆盖 ===")
# 创建临时目录
temp_dir = tempfile.mkdtemp()
try:
target_dir = os.path.join(temp_dir, 'public_kb')
os.makedirs(target_dir, exist_ok=True)
# 创建旧文件
old_file = os.path.join(target_dir, '规章制度.pdf')
with open(old_file, 'w', encoding='utf-8') as f:
f.write('旧内容')
assert os.path.exists(old_file), "旧文件应存在"
old_size = os.path.getsize(old_file)
print(f" [准备] 创建旧文件: {old_file}, 大小 {old_size} 字节")
# 模拟上传逻辑：同名文件覆盖
filename = '规章制度.pdf'
filepath = os.path.join(target_dir, filename)
replaced = False
if os.path.exists(filepath):
replaced = True
# 这里应该调用 kb_manager 清理旧切片，简化测试只验证文件覆盖
print(f" [检测] 发现同名文件，准备覆盖")
# 保存新文件（覆盖）
with open(filepath, 'w', encoding='utf-8') as f:
f.write('新内容，更长一些')
assert replaced, "应检测到同名文件并标记 replaced=True"
assert os.path.exists(filepath), "新文件应存在"
new_size = os.path.getsize(filepath)
assert new_size > old_size, f"新文件应更大（{new_size} > {old_size}）"
with open(filepath, 'r', encoding='utf-8') as f:
content = f.read()
assert content == '新内容，更长一些', "文件内容应为新内容"
print(f" [PASS] 文件已覆盖: {old_size} -> {new_size} 字节")
print(f" [PASS] replaced={replaced}")
finally:
shutil.rmtree(temp_dir)
def test_batch_upload_overwrite():
"""测试批量上传中的同名文件覆盖逻辑"""
print("\n=== 测试 3：批量上传同名文件覆盖 ===")
temp_dir = tempfile.mkdtemp()
try:
target_dir = os.path.join(temp_dir, 'public_kb')
os.makedirs(target_dir, exist_ok=True)
# 创建旧文件
old_file = os.path.join(target_dir, '规章制度.pdf')
with open(old_file, 'w', encoding='utf-8') as f:
f.write('旧内容')
# 模拟批量上传逻辑
files = [
('规章制度.pdf', '新内容1'),
('操作手册.pdf', '手册内容'),
]
results = []
for filename, content in files:
filepath = os.path.join(target_dir, filename)
replaced = False
if os.path.exists(filepath):
replaced = True
print(f" [检测] {filename} 已存在，准备覆盖")
with open(filepath, 'w', encoding='utf-8') as f:
f.write(content)
results.append({
'filename': filename,
'replaced': replaced
})
# 验证结果
assert len(results) == 2, f"应处理 2 个文件，实际 {len(results)}"
assert results[0]['replaced'] is True, "规章制度.pdf 应标记为 replaced"
assert results[1]['replaced'] is False, "操作手册.pdf 不应标记为 replaced"
print(f" [PASS] 批量上传: {len(results)} 个文件")
print(f" [PASS] 规章制度.pdf: replaced={results[0]['replaced']}")
print(f" [PASS] 操作手册.pdf: replaced={results[1]['replaced']}")
# 验证文件内容
with open(old_file, 'r', encoding='utf-8') as f:
content = f.read()
assert content == '新内容1', "规章制度.pdf 应为新内容"
print(f" [PASS] 文件内容已更新")
finally:
shutil.rmtree(temp_dir)
def test_docstore_cleanup():
"""测试 DocStore 文件清理逻辑"""
print("\n=== 测试 4：DocStore 文件清理 ===")
temp_dir = tempfile.mkdtemp()
try:
docstore_dir = Path(temp_dir) / 'docstore'
docstore_dir.mkdir()
# 创建模拟的 DocStore 文件
collection = 'public_kb'
filename = '规章制度.pdf'
# 旧切片对应的 DocStore 文件
old_files = [
f'{collection}_{filename}_0.json',
f'{collection}_{filename}_1.json',
f'{collection}_{filename}_2.json',
]
# 其他文件的 DocStore（不应被清理）
other_files = [
f'{collection}_操作手册.pdf_0.json',
f'dept_1_kb_{filename}_0.json', # 不同 collection
]
for f in old_files + other_files:
(docstore_dir / f).write_text('{"test": "data"}', encoding='utf-8')
assert len(list(docstore_dir.glob('*.json'))) == 5, "应有 5 个 DocStore 文件"
print(f" [准备] 创建 5 个 DocStore 文件")
# 模拟清理逻辑
cleaned = 0
for ds_file in docstore_dir.glob(f'{collection}_{filename}_*.json'):
ds_file.unlink()
cleaned += 1
assert cleaned == 3, f"应清理 3 个旧 DocStore 文件，实际 {cleaned}"
remaining = list(docstore_dir.glob('*.json'))
assert len(remaining) == 2, f"应剩余 2 个 DocStore 文件，实际 {len(remaining)}"
print(f" [PASS] 清理了 {cleaned} 个旧 DocStore 文件")
print(f" [PASS] 剩余 {len(remaining)} 个无关文件未被清理")
finally:
shutil.rmtree(temp_dir)
def test_sync_hash_cleanup():
"""测试同步哈希记录清理逻辑"""
print("\n=== 测试 5：同步哈希记录清理 ===")
# 模拟 SyncDatabase
class MockSyncDatabase:
def __init__(self):
self.hashes = {}
def set_document_hash(self, doc_id, doc_name, hash_val, size, mtime):
self.hashes[doc_id] = {
'document_id': doc_id,
'document_name': doc_name,
'content_hash': hash_val,
'file_size': size,
'last_modified': mtime
}
def get_document_hash(self, doc_id):
return self.hashes.get(doc_id)
def delete_document_hash(self, doc_id):
self.hashes.pop(doc_id, None)
db = MockSyncDatabase()
# 添加旧哈希记录
db.set_document_hash('public_kb/规章制度.pdf', '规章制度.pdf', 'old_hash_abc', 1024, '2026-01-01')
assert db.get_document_hash('public_kb/规章制度.pdf') is not None, "旧哈希应存在"
print(f" [准备] 创建旧哈希记录: public_kb/规章制度.pdf")
# 模拟上传时清理哈希
collection = 'public_kb'
filename = '规章制度.pdf'
db.delete_document_hash(f"{collection}/{filename}")
assert db.get_document_hash('public_kb/规章制度.pdf') is None, "旧哈希应已被清理"
print(f" [PASS] 哈希记录已清理")
def run_all_tests():
"""运行所有测试"""
print("=" * 50)
print("Phase 3 验证测试：重复上传旧切片残留修复")
print("=" * 50)
tests = [
test_add_file_to_kb_dedup,
test_upload_document_overwrite,
test_batch_upload_overwrite,
test_docstore_cleanup,
test_sync_hash_cleanup,
]
passed = 0
failed = 0
for test in tests:
try:
test()
passed += 1
except Exception as e:
print(f" [FAIL] {test.__name__}: {e}")
import traceback
traceback.print_exc()
failed += 1
print("\n" + "=" * 50)
print(f"测试结果：{passed} 通过 / {failed} 失败 / {len(tests)} 总计")
if failed == 0:
print("全部通过！")
else:
print(f"有 {failed} 个测试失败")
print("=" * 50)
return failed == 0
if __name__ == '__main__':
success = run_all_tests()
sys.exit(0 if success else 1)

437
tests/test_version_management.py Normal file
View File

@@ -0,0 +1,437 @@
# -*- coding: utf-8 -*-
"""
文档版本管理修复验证测试
测试覆盖：
1. mark_document_as_superseded 只更新 SQLite（不操作 ChromaDB）
2. deprecate_document 同步 SQLite 版本记录
3. restore_document 同步 SQLite 版本记录
4. 版本历史查询一致性
5. cleanup_superseded_versions 清理 SQLite 记录
6. 上传覆盖创建版本记录（逻辑验证）
"""
import sys
import os
import sqlite3
import tempfile
import shutil
from datetime import datetime, timedelta
from unittest.mock import patch, MagicMock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
passed = 0
failed = 0
def check(name, condition, detail=""):
global passed, failed
if condition:
print(f" [PASS] {name}")
passed += 1
else:
print(f" [FAIL] {name} {detail}")
failed += 1
# ======================================================================
# 辅助：创建临时 SQLite 数据库
# ======================================================================
def create_test_db():
"""创建内存中的测试数据库"""
conn = sqlite3.connect(":memory:")
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS document_versions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
document_id TEXT NOT NULL,
collection TEXT,
version TEXT NOT NULL DEFAULT 'v1',
content_hash TEXT,
status TEXT NOT NULL DEFAULT 'active',
effective_date DATE,
expiry_date DATE,
deprecated_date DATETIME,
deprecated_reason TEXT,
deprecated_by TEXT,
change_summary TEXT,
changed_sections TEXT,
supersedes TEXT,
chunk_count INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by TEXT,
UNIQUE(document_id, collection, version)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS version_change_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
document_id TEXT NOT NULL,
collection TEXT,
old_version TEXT,
new_version TEXT,
old_status TEXT,
new_status TEXT,
change_type TEXT NOT NULL,
reason TEXT,
changed_by TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
conn.commit()
return conn
def insert_version(conn, collection, doc_id, version, status="active"):
"""插入版本记录"""
conn.execute(
"""INSERT OR REPLACE INTO document_versions
(document_id, collection, version, status, created_at)
VALUES (?, ?, ?, ?, ?)""",
(doc_id, collection, version, status, datetime.now().isoformat())
)
conn.commit()
def get_versions(conn, collection, doc_id):
"""查询版本记录"""
cursor = conn.execute(
"""SELECT version, status, deprecated_date, deprecated_reason
FROM document_versions
WHERE collection = ? AND document_id = ?
ORDER BY created_at DESC""",
(collection, doc_id)
)
return cursor.fetchall()
def get_change_logs(conn, collection, doc_id):
"""查询变更日志"""
cursor = conn.execute(
"""SELECT change_type, old_version, new_version, old_status, new_status
FROM version_change_logs
WHERE collection = ? AND document_id = ?
ORDER BY created_at DESC""",
(collection, doc_id)
)
return cursor.fetchall()
# ======================================================================
# 测试 1：mark_document_as_superseded 参数签名验证
# ======================================================================
print("\n=== 测试 1：mark_document_as_superseded 新签名 ===")
from knowledge.manager import KnowledgeBaseManager
import inspect
sig = inspect.signature(KnowledgeBaseManager.mark_document_as_superseded)
params = list(sig.parameters.keys())
check("新签名包含 filename 参数",
"filename" in params,
f"参数列表: {params}")
check("新签名包含 new_version 参数",
"new_version" in params,
f"参数列表: {params}")
check("旧参数 old_filename 已移除",
"old_filename" not in params,
f"参数列表: {params}")
check("旧参数 new_filename 已移除",
"new_filename" not in params,
f"参数列表: {params}")
# ======================================================================
# 测试 2：deprecate_document 同步 SQLite（mock 验证）
# ======================================================================
print("\n=== 测试 2：deprecate_document SQLite 同步逻辑 ===")
from knowledge.document import DocumentMixin
class MockDocumentManager(DocumentMixin):
"""模拟文档管理器"""
def __init__(self):
self.mock_collection = MagicMock()
self._bm25_indexes = {}
def get_collection(self, kb_name):
return self.mock_collection
def rebuild_bm25_index(self, kb_name):
pass
# 模拟场景：3 个 chunk，active 状态
mgr = MockDocumentManager()
mgr.mock_collection.get.return_value = {
'ids': ['rule_0', 'rule_1', 'rule_2'],
'metadatas': [
{'source': 'rule.txt', 'status': 'active', 'version': 'v1'},
{'source': 'rule.txt', 'status': 'active', 'version': 'v1'},
{'source': 'rule.txt', 'status': 'active', 'version': 'v1'},
]
}
# 模拟 SQLite 连接
test_conn = create_test_db()
insert_version(test_conn, "test_kb", "rule.txt", "v1", "active")
with patch('data.db.get_connection', return_value=test_conn):
with patch('knowledge.document_versions.get_version_query') as mock_vq:
mock_vq_inst = MagicMock()
mock_vq_inst.get_active_version.return_value = MagicMock(version="v1")
mock_vq.return_value = mock_vq_inst
result = mgr.deprecate_document("test_kb", "rule.txt", reason="test deprecate")
check("deprecate 返回 success",
result.get("success") is True,
f"result: {result}")
check("deprecate 标记 3 个 chunks",
result.get("deprecated_chunks") == 3,
f"deprecated_chunks: {result.get('deprecated_chunks')}")
# 验证 SQLite 中的状态是否更新
versions = get_versions(test_conn, "test_kb", "rule.txt")
v1_status = versions[0][1] if versions else None
check("SQLite v1 状态改为 deprecated",
v1_status == "deprecated",
f"status: {v1_status}")
v1_deprecated_date = versions[0][2] if versions else None
check("SQLite deprecated_date 有值",
v1_deprecated_date is not None,
f"deprecated_date: {v1_deprecated_date}")
# ======================================================================
# 测试 3：restore_document 同步 SQLite（mock 验证）
# ======================================================================
print("\n=== 测试 3：restore_document SQLite 同步逻辑 ===")
mgr2 = MockDocumentManager()
mgr2.mock_collection.get.return_value = {
'ids': ['rule_0', 'rule_1'],
'metadatas': [
{'source': 'rule.txt', 'status': 'deprecated'},
{'source': 'rule.txt', 'status': 'deprecated'},
]
}
with patch('data.db.get_connection', return_value=test_conn):
with patch('knowledge.document_versions.get_version_query') as mock_vq:
mock_vq_inst = MagicMock()
mock_vq_inst.get_document_history.return_value = [
MagicMock(status=MagicMock(value='deprecated'), version='v1')
]
mock_vq.return_value = mock_vq_inst
result = mgr2.restore_document("test_kb", "rule.txt")
check("restore 返回 success",
result.get("success") is True,
f"result: {result}")
check("restore 恢复 2 个 chunks",
result.get("restored_chunks") == 2,
f"restored_chunks: {result.get('restored_chunks')}")
versions = get_versions(test_conn, "test_kb", "rule.txt")
v1_status = versions[0][1] if versions else None
check("SQLite v1 状态恢复为 active",
v1_status == "active",
f"status: {v1_status}")
# ======================================================================
# 测试 4：cleanup_superseded_versions 清理 SQLite 记录
# ======================================================================
print("\n=== 测试 4：cleanup_superseded_versions SQLite 清理 ===")
from knowledge.cleanup import cleanup_superseded_versions
test_conn2 = create_test_db()
# 插入一条过期的 superseded 记录
old_date = (datetime.now() - timedelta(days=30)).isoformat()
test_conn2.execute(
"""INSERT INTO document_versions
(document_id, collection, version, status, deprecated_date)
VALUES (?, ?, ?, ?, ?)""",
("old_report.pdf", "public_kb", "v1", "superseded", old_date)
)
# 插入一条近期的 superseded 记录
recent_date = (datetime.now() - timedelta(days=2)).isoformat()
test_conn2.execute(
"""INSERT INTO document_versions
(document_id, collection, version, status, deprecated_date)
VALUES (?, ?, ?, ?, ?)""",
("new_report.pdf", "public_kb", "v1", "superseded", recent_date)
)
# 插入一条旧日志
test_conn2.execute(
"""INSERT INTO version_change_logs
(document_id, collection, change_type, created_at)
VALUES (?, ?, ?, ?)""",
("old_report.pdf", "public_kb", "supersede", old_date)
)
test_conn2.commit()
with patch('data.db.get_connection', return_value=test_conn2):
cleaned = cleanup_superseded_versions(days_to_keep=7)
check("清理了 1 条过期 superseded 记录",
cleaned == 1,
f"cleaned: {cleaned}")
# 验证剩余记录
remaining = test_conn2.execute(
"SELECT document_id, status FROM document_versions"
).fetchall()
check("近期 superseded 记录保留",
any(r[0] == "new_report.pdf" and r[1] == "superseded" for r in remaining),
f"remaining: {remaining}")
check("过期 superseded 记录已删除",
not any(r[0] == "old_report.pdf" for r in remaining),
f"remaining: {remaining}")
# 验证日志也被清理
logs_remaining = test_conn2.execute(
"SELECT COUNT(*) FROM version_change_logs"
).fetchone()[0]
check("过期变更日志已清理",
logs_remaining == 0,
f"logs_remaining: {logs_remaining}")
# ======================================================================
# 测试 5：_filter_deprecated_chunks 过滤逻辑不变
# ======================================================================
print("\n=== 测试 5：_filter_deprecated_chunks 过滤逻辑 ===")
from core.engine import RAGEngine
# 模拟搜索结果
mock_results = {
'ids': [['id1', 'id2', 'id3', 'id4']],
'documents': [['doc1', 'doc2', 'doc3', 'doc4']],
'metadatas': [[
{'source': 'a.txt', 'status': 'active'},
{'source': 'b.txt', 'status': 'deprecated'},
{'source': 'c.txt'}, # 无 status 字段，默认 active
{'source': 'd.txt', 'status': 'superseded'},
]],
'distances': [[0.1, 0.2, 0.3, 0.4]],
}
# 直接调用实例方法（传 None 作为 self，方法中未使用 self）
filter_fn = RAGEngine._filter_deprecated_chunks
filtered = filter_fn(None, mock_results)
check("过滤后保留 2 条（active + 无status）",
len(filtered['ids'][0]) == 2,
f"ids: {filtered['ids'][0]}")
check("保留的 id 是 id1 和 id3",
filtered['ids'][0] == ['id1', 'id3'],
f"ids: {filtered['ids'][0]}")
check("deprecated 被过滤",
'id2' not in filtered['ids'][0],
"")
check("superseded 被过滤",
'id4' not in filtered['ids'][0],
"")
# ======================================================================
# 测试 6：upload 覆盖版本记录逻辑验证
# ======================================================================
print("\n=== 测试 6：upload 覆盖 + sync 版本记录逻辑（代码路径验证）===")
# 验证 document_routes.py 中 replaced 分支有 superseded 标记代码
routes_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"api", "document_routes.py"
)
with open(routes_file, 'r', encoding='utf-8') as f:
source = f.read()
check("upload_document 中有 get_version_query 调用",
"get_version_query" in source,
"")
check("upload_document 中有 superseded 状态更新",
"status='superseded'" in source,
"")
check("upload_document replaced 分支标记旧版本",
"重新上传覆盖" in source,
"")
# 验证 sync.py ADDED 分支使用自动版本号（不硬编码 v1）
sync_file2 = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"knowledge", "sync.py"
)
with open(sync_file2, 'r', encoding='utf-8') as f:
sync_src = f.read()
check("sync ADDED 分支使用 _generate_version_id",
"_generate_version_id" in sync_src,
"")
check("sync ADDED 版本记录使用动态版本号",
"version=new_version" in sync_src,
"")
# ======================================================================
# 测试 7：sync.py MODIFIED 流程调用正确性
# ======================================================================
print("\n=== 测试 7：sync.py MODIFIED 流程参数验证 ===")
sync_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"knowledge", "sync.py"
)
with open(sync_file, 'r', encoding='utf-8') as f:
sync_source = f.read()
check("MODIFIED 分支中 mark 调用包含 new_version 参数",
"new_version=new_version" in sync_source,
"")
# 在 MODIFIED 分支内验证调用顺序：先获取 old_version 再生成 new_version
modified_marker = "elif change.change_type == ChangeType.MODIFIED:"
mod_start = sync_source.index(modified_marker)
modified_section = sync_source[mod_start:mod_start + 2000]
check("MODIFIED 分支先获取 old_version 再生成 new_version",
"_get_current_version" in modified_section and
"_generate_version_id" in modified_section and
modified_section.index("_get_current_version") < modified_section.index("_generate_version_id"),
f"section contains: _get_current_version={'_get_current_version' in modified_section}, _generate_version_id={'_generate_version_id' in modified_section}")
check("MODIFIED 分支有 create_version_record 调用",
"create_version_record" in sync_source,
"")
# ======================================================================
# 汇总
# ======================================================================
print(f"\n{'='*60}")
print(f"测试完成: {passed} 通过, {failed} 失败, 共 {passed + failed} 条")
print(f"{'='*60}")
if failed > 0:
sys.exit(1)