0f339a0998
基于缓存审查报告 + Claude Code 交叉审查的反馈： P0+ 语义缓存版本失效: - sync.py: 文档变更后清空语义缓存，防止返回过时的 images/sources - document_routes.py: 删除文档时同步清理缓存 - kb_routes.py: 删除向量库时同步清理缓存 P1.5 cache_type 包含式校验: - intent_analyzer.py: != "rag_answer" 改为 == "intent_analysis" P2+ Rerank Cache 版本失效: - cache.py: increment_kb_version 时同时清空 rerank_cache P1+P6 死代码清理: - agentic.py: 移除未使用的 SemanticCache 实例和相关导入 - agentic_base.py: 移除未使用的语义缓存配置导入 - cache.py: 移除 _compute_doc_hash 和未使用的 doc_ids 参数 - engine.py: 移除 set_query_result 调用中的 doc_ids 参数 P7 缓存质量门槛: - config.py: CACHE_MIN_SCORE 从 0.0 改为 0.3
906 lines
27 KiB
Python
906 lines
27 KiB
Python
"""
|
||
多向量库管理 API
|
||
|
||
本模块提供向量库的 CRUD 操作和文档同步功能，支持：
|
||
- 创建、查询、修改、删除向量库
|
||
- 文档向量化同步
|
||
- 文档版本管理（废止/恢复）
|
||
- 知识库路由测试
|
||
|
||
路由列表:
|
||
GET /collections : 获取向量库列表
|
||
POST /collections : 创建向量库
|
||
PUT /collections/<kb_name> : 修改向量库信息
|
||
DELETE /collections/<kb_name> : 删除向量库
|
||
GET /collections/<kb_name>/documents : 获取向量库文档列表
|
||
GET /collections/<kb_name>/chunks : 获取向量库切片列表
|
||
POST /documents/sync : 触发文档同步
|
||
POST /kb/route : 测试知识库路由（调试）
|
||
|
||
文档版本管理:
|
||
POST /collections/<kb_name>/documents/<path:filename>/deprecate : 废止文档
|
||
POST /collections/<kb_name>/documents/<path:filename>/restore : 恢复文档
|
||
GET /collections/<kb_name>/documents/<path:filename>/versions : 版本历史
|
||
|
||
架构说明:
|
||
- 权限验证由后端网关完成，RAG 服务不做权限判断
|
||
- 每个部门可拥有独立的向量库（多租户架构）
|
||
- 向量库元数据存储在 kb_metadata.json
|
||
|
||
Example:
|
||
curl -X GET http://localhost:5001/collections \\
|
||
-H "Authorization: Bearer mock-token-admin"
|
||
"""
|
||
|
||
import os
|
||
from typing import Tuple, Optional, Any
|
||
from flask import Blueprint, request, jsonify, current_app
|
||
import logging
|
||
|
||
logger = logging.getLogger(__name__)
|
||
from auth.gateway import require_gateway_auth
|
||
|
||
kb_bp = Blueprint('kb', __name__)
|
||
|
||
# 延迟初始化缓存
|
||
_kb_manager = None
|
||
_kb_router = None
|
||
_kb_checked = False
|
||
_has_multi_kb = False
|
||
|
||
|
||
def _check_multi_kb() -> bool:
|
||
"""
|
||
检查多向量库模块是否可用
|
||
|
||
延迟导入知识库管理器和路由器，避免循环依赖。
|
||
结果缓存在模块级变量中，后续调用直接返回缓存。
|
||
|
||
Returns:
|
||
多向量库可用返回 True，否则返回 False
|
||
"""
|
||
global _kb_checked, _has_multi_kb, _kb_manager, _kb_router
|
||
if not _kb_checked:
|
||
try:
|
||
from knowledge.manager import get_kb_manager
|
||
from knowledge.router import get_kb_router
|
||
_kb_manager = get_kb_manager()
|
||
_kb_router = get_kb_router()
|
||
_has_multi_kb = True
|
||
except ImportError as e:
|
||
logger.warning(f"多向量库模块导入失败: {e}")
|
||
_has_multi_kb = False
|
||
_kb_checked = True
|
||
return _has_multi_kb
|
||
|
||
|
||
def _require_multi_kb() -> Tuple[Optional[Any], Optional[Any], Optional[Tuple]]:
|
||
"""
|
||
检查多向量库是否可用，返回管理器和路由器
|
||
|
||
Returns:
|
||
元组 (kb_manager, kb_router, error_response)：
|
||
- 成功时 error_response 为 None
|
||
- 失败时 kb_manager 和 kb_router 为 None
|
||
|
||
Example:
|
||
>>> kb_manager, kb_router, err = _require_multi_kb()
|
||
>>> if err:
|
||
... return err
|
||
"""
|
||
if not _check_multi_kb():
|
||
return None, None, (jsonify({"error": "多向量库模块未启用"}), 503)
|
||
return _kb_manager, _kb_router, None
|
||
|
||
|
||
@kb_bp.route('/collections', methods=['GET'])
|
||
@require_gateway_auth
|
||
def list_collections() -> Tuple[Any, int]:
|
||
"""
|
||
获取向量库列表
|
||
|
||
返回当前用户可访问的所有向量库信息，包括：
|
||
- 向量库名称和显示名称
|
||
- 文档数量
|
||
- 所属部门
|
||
- 创建时间
|
||
- 描述信息
|
||
|
||
Returns:
|
||
JSON 响应，包含 collections 列表和 total 总数
|
||
|
||
Example:
|
||
GET /collections
|
||
Response: {"collections": [...], "total": 3}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
user = request.current_user
|
||
|
||
# 获取所有向量库（权限由后端网关管理）
|
||
all_collections = kb_manager.list_collections()
|
||
|
||
result = []
|
||
for coll in all_collections:
|
||
result.append({
|
||
"name": coll.name,
|
||
"display_name": coll.display_name,
|
||
"document_count": coll.document_count,
|
||
"department": coll.department,
|
||
"created_at": coll.created_at,
|
||
"description": coll.description
|
||
})
|
||
|
||
return jsonify({
|
||
"collections": result,
|
||
"total": len(result)
|
||
})
|
||
|
||
|
||
@kb_bp.route('/collections', methods=['POST'])
|
||
@require_gateway_auth
|
||
def create_collection() -> Tuple[Any, int]:
|
||
"""
|
||
创建新向量库
|
||
|
||
请求体:
|
||
{
|
||
"name": "向量库标识（英文）",
|
||
"display_name": "显示名称",
|
||
"department": "所属部门",
|
||
"description": "描述信息"
|
||
}
|
||
|
||
注意:
|
||
- 名称只能包含字母、数字、下划线和连字符
|
||
- 名称不能为空
|
||
|
||
Returns:
|
||
成功: {"success": true, "message": "...", "name": "..."} (201)
|
||
失败: {"error": "..."} (400)
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
data = request.json or {}
|
||
name = data.get('name', '').strip()
|
||
display_name = data.get('display_name', '')
|
||
department = data.get('department', '')
|
||
description = data.get('description', '')
|
||
|
||
if not name:
|
||
return jsonify({"error": "向量库名称不能为空"}), 400
|
||
|
||
# 验证名称格式（ChromaDB 限制）
|
||
if not name.replace('_', '').replace('-', '').isalnum():
|
||
return jsonify({
|
||
"error": "名称格式错误",
|
||
"message": "向量库名称只能包含字母、数字、下划线和连字符"
|
||
}), 400
|
||
|
||
success, message = kb_manager.create_collection(
|
||
name, display_name, department, description
|
||
)
|
||
|
||
if success:
|
||
return jsonify({"success": True, "message": message, "name": name}), 201
|
||
return jsonify({"error": message}), 400
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>', methods=['PUT'])
|
||
@require_gateway_auth
|
||
def update_collection(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
修改向量库信息
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
请求体:
|
||
{
|
||
"display_name": "新显示名称",
|
||
"description": "新描述"
|
||
}
|
||
|
||
Returns:
|
||
成功: {"success": true, "message": "向量库信息已更新"}
|
||
失败: {"error": "..."} (404/500)
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
data = request.json or {}
|
||
display_name = data.get('display_name')
|
||
description = data.get('description')
|
||
|
||
# 检查向量库是否存在
|
||
collections = kb_manager.list_collections()
|
||
if not any(c.name == kb_name for c in collections):
|
||
return jsonify({"error": f"向量库 '{kb_name}' 不存在"}), 404
|
||
|
||
# 更新元数据
|
||
success = kb_manager.update_collection_metadata(
|
||
kb_name,
|
||
display_name=display_name,
|
||
description=description
|
||
)
|
||
|
||
if success:
|
||
return jsonify({"success": True, "message": "向量库信息已更新"})
|
||
return jsonify({"error": "更新失败"}), 500
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>', methods=['DELETE'])
|
||
@require_gateway_auth
|
||
def delete_collection(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
删除向量库
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
查询参数:
|
||
delete_documents: 是否删除文档源文件（默认 false）
|
||
|
||
Returns:
|
||
成功: {"success": true, "message": "...", "deleted_documents": bool}
|
||
失败: {"error": "..."} (400)
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
# 获取参数
|
||
delete_documents = request.args.get('delete_documents', 'false').lower() == 'true'
|
||
|
||
success, message = kb_manager.delete_collection(kb_name, delete_documents)
|
||
|
||
if success:
|
||
# 缓存失效
|
||
try:
|
||
from core.cache import get_cache_manager
|
||
_cm = get_cache_manager()
|
||
_cm.increment_kb_version(kb_name)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
from core.semantic_cache import get_semantic_cache
|
||
_sc = get_semantic_cache()
|
||
if _sc:
|
||
_sc.clear()
|
||
except Exception:
|
||
pass
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"message": message,
|
||
"deleted_documents": delete_documents
|
||
})
|
||
return jsonify({"error": message}), 400
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/documents', methods=['GET'])
|
||
@require_gateway_auth
|
||
def list_collection_documents(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
获取向量库中的文档列表
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
Returns:
|
||
{"collection": "...", "documents": [...], "total": N}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
documents = kb_manager.list_documents(kb_name)
|
||
|
||
return jsonify({
|
||
"collection": kb_name,
|
||
"documents": documents,
|
||
"total": len(documents)
|
||
})
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/chunks', methods=['GET'])
|
||
@require_gateway_auth
|
||
def list_collection_chunks(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
获取向量库中的切片列表
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
查询参数:
|
||
document_id: 过滤指定文档的切片（可选）
|
||
limit: 返回数量限制（默认 100）
|
||
offset: 偏移量（默认 0）
|
||
|
||
Returns:
|
||
{"collection": "...", "chunks": [...], "total": N}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
# 可选过滤参数
|
||
document_id = request.args.get('document_id')
|
||
limit = request.args.get('limit', 100, type=int)
|
||
offset = request.args.get('offset', 0, type=int)
|
||
|
||
chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset)
|
||
|
||
return jsonify({
|
||
"collection": kb_name,
|
||
"chunks": chunks,
|
||
"total": len(chunks)
|
||
})
|
||
|
||
|
||
@kb_bp.route('/documents/sync', methods=['POST'])
|
||
@require_gateway_auth
|
||
def sync_documents() -> Tuple[Any, int]:
|
||
"""
|
||
触发文档向量化同步
|
||
|
||
扫描文档目录，检测新增、修改、删除的文件，
|
||
自动更新向量库索引。
|
||
|
||
请求体:
|
||
{
|
||
"collection": "向量库名称" // 可选，不传则同步所有
|
||
}
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"results": [{"collection": "...", "status": "...", ...}],
|
||
"synced_count": N
|
||
}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
from config import DOCUMENTS_PATH
|
||
|
||
user = request.current_user
|
||
data = request.json or {}
|
||
target_collection = data.get('collection')
|
||
|
||
# 确定要同步的向量库
|
||
if target_collection:
|
||
collections_to_sync = [target_collection]
|
||
else:
|
||
# 同步所有向量库
|
||
all_collections = kb_manager.list_collections()
|
||
collections_to_sync = [c.name for c in all_collections]
|
||
|
||
if not collections_to_sync:
|
||
return jsonify({"error": "没有可同步的向量库"}), 400
|
||
|
||
# 执行同步
|
||
results = []
|
||
|
||
# 使用 sync_service 执行同步
|
||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||
|
||
if sync_service:
|
||
try:
|
||
sync_result = sync_service.sync_now()
|
||
results.append({
|
||
"collection": "all",
|
||
"status": "success",
|
||
"message": f"同步完成: 处理 {sync_result.documents_processed} 个文档",
|
||
"details": {
|
||
"added": sync_result.documents_added,
|
||
"modified": sync_result.documents_modified,
|
||
"deleted": sync_result.documents_deleted,
|
||
"errors": sync_result.errors
|
||
}
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"知识库操作异常: {e}")
|
||
results.append({
|
||
"collection": "all",
|
||
"status": "error",
|
||
"message": "操作失败"
|
||
})
|
||
else:
|
||
# 没有 sync_service，返回提示
|
||
for coll_name in collections_to_sync:
|
||
results.append({
|
||
"collection": coll_name,
|
||
"status": "warning",
|
||
"message": "同步服务不可用，请使用 POST /sync 端点"
|
||
})
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"results": results,
|
||
"synced_count": len([r for r in results if r["status"] == "success"])
|
||
})
|
||
|
||
|
||
@kb_bp.route('/debug/scan', methods=['GET'])
|
||
@require_gateway_auth
|
||
def debug_scan() -> Tuple[Any, int]:
|
||
"""
|
||
调试：查看同步服务扫描的文件
|
||
|
||
返回文档目录的文件列表和同步服务状态，
|
||
用于排查文档同步问题。
|
||
|
||
Returns:
|
||
{
|
||
"documents_path": "...",
|
||
"exists": bool,
|
||
"files": [...],
|
||
"sync_service_path": "...",
|
||
"scanned_count": N
|
||
}
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
import os
|
||
|
||
result = {
|
||
"documents_path": DOCUMENTS_PATH,
|
||
"exists": os.path.exists(DOCUMENTS_PATH),
|
||
"files": []
|
||
}
|
||
|
||
if os.path.exists(DOCUMENTS_PATH):
|
||
for root, dirs, files in os.walk(DOCUMENTS_PATH):
|
||
for f in files:
|
||
fp = os.path.join(root, f)
|
||
rel = os.path.relpath(fp, DOCUMENTS_PATH)
|
||
result["files"].append({
|
||
"rel_path": rel,
|
||
"size": os.path.getsize(fp),
|
||
"ext": os.path.splitext(f)[1].lower()
|
||
})
|
||
|
||
# 也检查同步服务的路径
|
||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||
if sync_service:
|
||
result["sync_service_path"] = sync_service.documents_path
|
||
result["sync_service_path_exists"] = os.path.exists(sync_service.documents_path)
|
||
|
||
# 直接调用 scan_documents
|
||
try:
|
||
scanned = sync_service.scan_documents()
|
||
result["scanned_count"] = len(scanned)
|
||
result["scanned_ids"] = list(scanned.keys())[:10]
|
||
except Exception as e:
|
||
logger.error(f"扫描异常: {e}")
|
||
result["scan_error"] = "扫描失败"
|
||
|
||
return jsonify(result)
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/reindex', methods=['POST'])
|
||
@require_gateway_auth
|
||
def reindex_collection(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
强制重新向量化指定集合的所有文档
|
||
|
||
清除该集合的文档哈希记录，触发完整重新索引。
|
||
适用于文档内容更新后需要重建索引的场景。
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"message": "...",
|
||
"documents_processed": N,
|
||
"documents_added": N,
|
||
"errors": [...]
|
||
}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
from config import DOCUMENTS_PATH
|
||
|
||
# 清除该集合的文档哈希记录
|
||
try:
|
||
from data.db import get_connection
|
||
with get_connection("knowledge") as conn:
|
||
cursor = conn.cursor()
|
||
# 转义 LIKE 通配符，防止 kb_name 中的 % 或 _ 导致非预期匹配
|
||
escaped_kb = kb_name.replace('%', '\\%').replace('_', '\\_')
|
||
# 删除以 "{kb_name}/" 或 "{kb_name}\" 开头的文档哈希（兼容 Windows 和 Linux）
|
||
cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? ESCAPE '\\' OR document_id LIKE ? ESCAPE '\\'",
|
||
(f"{escaped_kb}/%", f"{escaped_kb}\\%"))
|
||
deleted = cursor.rowcount
|
||
logger.info(f"已清除 {deleted} 条哈希记录: {kb_name}")
|
||
except Exception as e:
|
||
logger.warning(f"清除哈希记录失败: {e}")
|
||
|
||
# 触发同步
|
||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||
if sync_service:
|
||
try:
|
||
result = sync_service.sync_now()
|
||
return jsonify({
|
||
"success": True,
|
||
"message": f"重新索引完成: 处理 {result.documents_processed} 个文档",
|
||
"documents_processed": result.documents_processed,
|
||
"documents_added": result.documents_added,
|
||
"errors": result.errors
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"reindex 异常: {e}")
|
||
return jsonify({"error": "操作失败，请稍后重试"}), 500
|
||
else:
|
||
return jsonify({"error": "同步服务不可用"}), 503
|
||
|
||
|
||
@kb_bp.route('/kb/route', methods=['POST'])
|
||
@require_gateway_auth
|
||
def test_routing() -> Tuple[Any, int]:
|
||
"""
|
||
测试知识库路由（调试用）
|
||
|
||
分析查询意图，返回目标向量库列表和意图分析结果。
|
||
|
||
请求体:
|
||
{"query": "用户查询"}
|
||
|
||
Returns:
|
||
{
|
||
"query": "...",
|
||
"user_role": "...",
|
||
"user_department": "...",
|
||
"target_collections": [...],
|
||
"intent": {
|
||
"is_general": bool,
|
||
"department": "...",
|
||
"confidence": float,
|
||
"keywords": [...],
|
||
"reason": "..."
|
||
}
|
||
}
|
||
"""
|
||
_, kb_router, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
from knowledge.router import route_query
|
||
|
||
user = request.current_user
|
||
data = request.json or {}
|
||
query = data.get('query', '')
|
||
|
||
if not query:
|
||
return jsonify({"error": "请提供查询内容"}), 400
|
||
|
||
# 获取路由结果
|
||
target_kbs = route_query(
|
||
query,
|
||
user.get("role", "user"),
|
||
user.get("department", "")
|
||
)
|
||
|
||
# 获取意图分析
|
||
intent = kb_router.analyze_intent(query)
|
||
|
||
return jsonify({
|
||
"query": query,
|
||
"user_role": user.get("role"),
|
||
"user_department": user.get("department", ""),
|
||
"target_collections": target_kbs,
|
||
"intent": {
|
||
"is_general": intent.is_general,
|
||
"department": intent.department,
|
||
"confidence": intent.confidence,
|
||
"keywords": intent.keywords,
|
||
"reason": intent.reason
|
||
}
|
||
})
|
||
|
||
|
||
# ==================== 文档版本管理 API ====================
|
||
|
||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/deprecate', methods=['POST'])
|
||
@require_gateway_auth
|
||
def deprecate_document(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||
"""
|
||
废止文档（软删除）
|
||
|
||
将文档标记为已废止，相关切片在检索时被过滤。
|
||
文档数据保留，可通过 restore_document 恢复。
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
filename: 文档文件名
|
||
|
||
请求体:
|
||
{"reason": "废止原因"}
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"deprecated_chunks": N,
|
||
"document_id": "...",
|
||
"collection": "...",
|
||
"deprecated_date": "ISO 8601 时间"
|
||
}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
user = request.current_user
|
||
data = request.json or {}
|
||
reason = data.get('reason', '文档已废止')
|
||
|
||
try:
|
||
result = kb_manager.deprecate_document(
|
||
kb_name,
|
||
filename,
|
||
reason,
|
||
deprecated_by=user.get('user_id', 'unknown')
|
||
)
|
||
return jsonify(result)
|
||
except Exception as e:
|
||
logger.error(f"操作异常: {e}")
|
||
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/restore', methods=['POST'])
|
||
@require_gateway_auth
|
||
def restore_document(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||
"""
|
||
恢复已废止的文档
|
||
|
||
将已废止的文档恢复为有效状态，相关切片重新参与检索。
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
filename: 文档文件名
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"restored_chunks": N,
|
||
"document_id": "...",
|
||
"collection": "..."
|
||
}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
try:
|
||
result = kb_manager.restore_document(kb_name, filename)
|
||
return jsonify(result)
|
||
except Exception as e:
|
||
logger.error(f"操作异常: {e}")
|
||
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/versions', methods=['GET'])
|
||
@require_gateway_auth
|
||
def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||
"""
|
||
获取文档版本历史
|
||
|
||
返回文档的所有版本记录，包括状态变更、时间戳等信息。
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
filename: 文档文件名
|
||
|
||
查询参数:
|
||
limit: 返回数量限制（默认 10）
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"document_id": "...",
|
||
"collection": "...",
|
||
"versions": [
|
||
{
|
||
"version": "v2",
|
||
"status": "active",
|
||
"created_at": "ISO 8601",
|
||
"chunk_count": N
|
||
},
|
||
...
|
||
],
|
||
"total": N
|
||
}
|
||
"""
|
||
_, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
limit = request.args.get('limit', 10, type=int)
|
||
|
||
try:
|
||
from knowledge.document_versions import get_version_query
|
||
version_query = get_version_query()
|
||
|
||
versions = version_query.get_document_history(kb_name, filename, limit)
|
||
versions_data = [v.to_dict() for v in versions]
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"document_id": filename,
|
||
"collection": kb_name,
|
||
"versions": versions_data,
|
||
"total": len(versions_data)
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"操作异常: {e}")
|
||
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
|
||
|
||
|
||
@kb_bp.route('/collections/<kb_name>/update-image-descriptions', methods=['POST'])
|
||
@require_gateway_auth
|
||
def update_image_descriptions(kb_name: str) -> Tuple[Any, int]:
|
||
"""
|
||
更新图片切片的轻量级描述
|
||
|
||
重新提取已入库文档中图片切片的图号/表号信息，
|
||
生成新的轻量级描述，提高检索准确度。
|
||
|
||
适用场景:
|
||
- 文档入库时未提取图号
|
||
- 图号提取规则更新后需要重新处理
|
||
|
||
Args:
|
||
kb_name: 向量库名称
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"image_count": N,
|
||
"updated_count": M
|
||
}
|
||
"""
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
try:
|
||
result = kb_manager.update_image_descriptions(kb_name)
|
||
return jsonify(result)
|
||
except Exception as e:
|
||
logger.error(f"操作异常: {e}")
|
||
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
|
||
|
||
|
||
@kb_bp.route('/collections/sync-vlm-cache', methods=['POST'])
|
||
@require_gateway_auth
|
||
def sync_vlm_cache() -> Tuple[Any, int]:
|
||
"""
|
||
同步 VLM 缓存到向量库元数据
|
||
|
||
将 .data/cache/vlm/ 中已生成的图片描述同步到向量库的 vlm_desc 字段。
|
||
同步后，检索时可直接从向量库获取 VLM 描述，无需读取文件系统。
|
||
|
||
请求参数:
|
||
- collection: 可选，指定同步的集合名，不传则同步所有集合
|
||
- dry_run: 可选，设为 true 只预览不执行
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"total_cache_files": N,
|
||
"synced_count": M,
|
||
"skipped_count": K,
|
||
"details": [...]
|
||
}
|
||
"""
|
||
from pathlib import Path
|
||
import hashlib
|
||
import json as json_module
|
||
|
||
kb_manager, _, err = _require_multi_kb()
|
||
if err:
|
||
return err
|
||
|
||
data = request.get_json() or {}
|
||
collection_filter = data.get('collection')
|
||
dry_run = data.get('dry_run', False)
|
||
|
||
vlm_cache_dir = Path(".data/cache/vlm")
|
||
images_dir = Path(".data/images")
|
||
|
||
if not vlm_cache_dir.exists():
|
||
return jsonify({"success": False, "error": "VLM 缓存目录不存在"}), 400
|
||
|
||
# 获取所有 VLM 缓存文件
|
||
cache_files = list(vlm_cache_dir.glob("*.txt"))
|
||
if not cache_files:
|
||
return jsonify({"success": True, "total_cache_files": 0, "synced_count": 0, "message": "无 VLM 缓存文件"})
|
||
|
||
# 构建图片 MD5 → 文件名 的映射
|
||
image_hash_map = {}
|
||
if images_dir.exists():
|
||
for img_file in images_dir.glob("*.png"):
|
||
img_hash = hashlib.md5(img_file.read_bytes()).hexdigest()
|
||
image_hash_map[img_hash] = img_file.name
|
||
|
||
# 结果统计
|
||
synced_count = 0
|
||
skipped_count = 0
|
||
details = []
|
||
|
||
# 获取所有集合
|
||
collections = kb_manager.list_collections()
|
||
if collection_filter:
|
||
collections = [c for c in collections if c.name == collection_filter]
|
||
|
||
for cache_file in cache_files:
|
||
cache_hash = cache_file.stem # 文件名即 MD5
|
||
|
||
# 查找对应的图片文件名
|
||
image_filename = image_hash_map.get(cache_hash)
|
||
if not image_filename:
|
||
skipped_count += 1
|
||
details.append({"cache": cache_file.name, "status": "skipped", "reason": "图片文件不存在"})
|
||
continue
|
||
|
||
# 读取 VLM 描述
|
||
vlm_desc = cache_file.read_text(encoding='utf-8')
|
||
|
||
# 在所有集合中查找该图片切片
|
||
found = False
|
||
for coll in collections:
|
||
collection = kb_manager.get_collection(coll.name)
|
||
if not collection:
|
||
continue
|
||
|
||
# 查找 image_path 匹配的切片
|
||
result = collection.get(where={"image_path": image_filename})
|
||
|
||
if result['ids']:
|
||
found = True
|
||
if not dry_run:
|
||
# 更新元数据
|
||
updated_metadatas = []
|
||
for meta in result['metadatas']:
|
||
meta['vlm_desc'] = vlm_desc
|
||
meta['has_vlm_desc'] = True
|
||
updated_metadatas.append(meta)
|
||
|
||
collection.update(
|
||
ids=result['ids'],
|
||
metadatas=updated_metadatas
|
||
)
|
||
|
||
synced_count += len(result['ids'])
|
||
details.append({
|
||
"cache": cache_file.name,
|
||
"image": image_filename,
|
||
"collection": coll.name,
|
||
"count": len(result['ids']),
|
||
"status": "synced" if not dry_run else "preview"
|
||
})
|
||
|
||
if not found:
|
||
skipped_count += 1
|
||
details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"})
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"total_cache_files": len(cache_files),
|
||
"synced_count": synced_count,
|
||
"skipped_count": skipped_count,
|
||
"dry_run": dry_run,
|
||
"details": details[:20] # 只返回前 20 条详情
|
||
})
|
||
|