init: RAG 知识库服务初始提交
- 后端 API（Flask + Gunicorn） - RAG 引擎（混合检索 + 云端 Reranker + 引用溯源） - 文档解析（MinerU + 多格式支持） - Docker 生产部署配置 - 排除前端项目、敏感配置、模型文件
This commit is contained in:
lacerate551
881
api/kb_routes.py
Normal file
881
api/kb_routes.py
Normal file
@@ -0,0 +1,881 @@
|
||||
"""
|
||||
多向量库管理 API
|
||||
|
||||
本模块提供向量库的 CRUD 操作和文档同步功能，支持：
|
||||
- 创建、查询、修改、删除向量库
|
||||
- 文档向量化同步
|
||||
- 文档版本管理（废止/恢复）
|
||||
- 知识库路由测试
|
||||
|
||||
路由列表:
|
||||
GET /collections : 获取向量库列表
|
||||
POST /collections : 创建向量库
|
||||
PUT /collections/<kb_name> : 修改向量库信息
|
||||
DELETE /collections/<kb_name> : 删除向量库
|
||||
GET /collections/<kb_name>/documents : 获取向量库文档列表
|
||||
GET /collections/<kb_name>/chunks : 获取向量库切片列表
|
||||
POST /documents/sync : 触发文档同步
|
||||
POST /kb/route : 测试知识库路由（调试）
|
||||
|
||||
文档版本管理:
|
||||
POST /collections/<kb_name>/documents/<path:filename>/deprecate : 废止文档
|
||||
POST /collections/<kb_name>/documents/<path:filename>/restore : 恢复文档
|
||||
GET /collections/<kb_name>/documents/<path:filename>/versions : 版本历史
|
||||
|
||||
架构说明:
|
||||
- 权限验证由后端网关完成，RAG 服务不做权限判断
|
||||
- 每个部门可拥有独立的向量库（多租户架构）
|
||||
- 向量库元数据存储在 kb_metadata.json
|
||||
|
||||
Example:
|
||||
curl -X GET http://localhost:5001/collections \\
|
||||
-H "Authorization: Bearer mock-token-admin"
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Tuple, Optional, Any
|
||||
from flask import Blueprint, request, jsonify, current_app
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from auth.gateway import require_gateway_auth
|
||||
|
||||
kb_bp = Blueprint('kb', __name__)
|
||||
|
||||
# 延迟初始化缓存
|
||||
_kb_manager = None
|
||||
_kb_router = None
|
||||
_kb_checked = False
|
||||
_has_multi_kb = False
|
||||
|
||||
|
||||
def _check_multi_kb() -> bool:
|
||||
"""
|
||||
检查多向量库模块是否可用
|
||||
|
||||
延迟导入知识库管理器和路由器，避免循环依赖。
|
||||
结果缓存在模块级变量中，后续调用直接返回缓存。
|
||||
|
||||
Returns:
|
||||
多向量库可用返回 True，否则返回 False
|
||||
"""
|
||||
global _kb_checked, _has_multi_kb, _kb_manager, _kb_router
|
||||
if not _kb_checked:
|
||||
try:
|
||||
from knowledge.manager import get_kb_manager
|
||||
from knowledge.router import get_kb_router
|
||||
_kb_manager = get_kb_manager()
|
||||
_kb_router = get_kb_router()
|
||||
_has_multi_kb = True
|
||||
except ImportError as e:
|
||||
logger.warning(f"多向量库模块导入失败: {e}")
|
||||
_has_multi_kb = False
|
||||
_kb_checked = True
|
||||
return _has_multi_kb
|
||||
|
||||
|
||||
def _require_multi_kb() -> Tuple[Optional[Any], Optional[Any], Optional[Tuple]]:
|
||||
"""
|
||||
检查多向量库是否可用，返回管理器和路由器
|
||||
|
||||
Returns:
|
||||
元组 (kb_manager, kb_router, error_response)：
|
||||
- 成功时 error_response 为 None
|
||||
- 失败时 kb_manager 和 kb_router 为 None
|
||||
|
||||
Example:
|
||||
>>> kb_manager, kb_router, err = _require_multi_kb()
|
||||
>>> if err:
|
||||
... return err
|
||||
"""
|
||||
if not _check_multi_kb():
|
||||
return None, None, (jsonify({"error": "多向量库模块未启用"}), 503)
|
||||
return _kb_manager, _kb_router, None
|
||||
|
||||
|
||||
@kb_bp.route('/collections', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def list_collections() -> Tuple[Any, int]:
|
||||
"""
|
||||
获取向量库列表
|
||||
|
||||
返回当前用户可访问的所有向量库信息，包括：
|
||||
- 向量库名称和显示名称
|
||||
- 文档数量
|
||||
- 所属部门
|
||||
- 创建时间
|
||||
- 描述信息
|
||||
|
||||
Returns:
|
||||
JSON 响应，包含 collections 列表和 total 总数
|
||||
|
||||
Example:
|
||||
GET /collections
|
||||
Response: {"collections": [...], "total": 3}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
user = request.current_user
|
||||
|
||||
# 获取所有向量库（权限由后端网关管理）
|
||||
all_collections = kb_manager.list_collections()
|
||||
|
||||
result = []
|
||||
for coll in all_collections:
|
||||
result.append({
|
||||
"name": coll.name,
|
||||
"display_name": coll.display_name,
|
||||
"document_count": coll.document_count,
|
||||
"department": coll.department,
|
||||
"created_at": coll.created_at,
|
||||
"description": coll.description
|
||||
})
|
||||
|
||||
return jsonify({
|
||||
"collections": result,
|
||||
"total": len(result)
|
||||
})
|
||||
|
||||
|
||||
@kb_bp.route('/collections', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def create_collection() -> Tuple[Any, int]:
|
||||
"""
|
||||
创建新向量库
|
||||
|
||||
请求体:
|
||||
{
|
||||
"name": "向量库标识（英文）",
|
||||
"display_name": "显示名称",
|
||||
"department": "所属部门",
|
||||
"description": "描述信息"
|
||||
}
|
||||
|
||||
注意:
|
||||
- 名称只能包含字母、数字、下划线和连字符
|
||||
- 名称不能为空
|
||||
|
||||
Returns:
|
||||
成功: {"success": true, "message": "...", "name": "..."} (201)
|
||||
失败: {"error": "..."} (400)
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
data = request.json or {}
|
||||
name = data.get('name', '').strip()
|
||||
display_name = data.get('display_name', '')
|
||||
department = data.get('department', '')
|
||||
description = data.get('description', '')
|
||||
|
||||
if not name:
|
||||
return jsonify({"error": "向量库名称不能为空"}), 400
|
||||
|
||||
# 验证名称格式（ChromaDB 限制）
|
||||
if not name.replace('_', '').replace('-', '').isalnum():
|
||||
return jsonify({
|
||||
"error": "名称格式错误",
|
||||
"message": "向量库名称只能包含字母、数字、下划线和连字符"
|
||||
}), 400
|
||||
|
||||
success, message = kb_manager.create_collection(
|
||||
name, display_name, department, description
|
||||
)
|
||||
|
||||
if success:
|
||||
return jsonify({"success": True, "message": message, "name": name}), 201
|
||||
return jsonify({"error": message}), 400
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>', methods=['PUT'])
|
||||
@require_gateway_auth
|
||||
def update_collection(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
修改向量库信息
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
请求体:
|
||||
{
|
||||
"display_name": "新显示名称",
|
||||
"description": "新描述"
|
||||
}
|
||||
|
||||
Returns:
|
||||
成功: {"success": true, "message": "向量库信息已更新"}
|
||||
失败: {"error": "..."} (404/500)
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
data = request.json or {}
|
||||
display_name = data.get('display_name')
|
||||
description = data.get('description')
|
||||
|
||||
# 检查向量库是否存在
|
||||
collections = kb_manager.list_collections()
|
||||
if not any(c.name == kb_name for c in collections):
|
||||
return jsonify({"error": f"向量库 '{kb_name}' 不存在"}), 404
|
||||
|
||||
# 更新元数据
|
||||
success = kb_manager.update_collection_metadata(
|
||||
kb_name,
|
||||
display_name=display_name,
|
||||
description=description
|
||||
)
|
||||
|
||||
if success:
|
||||
return jsonify({"success": True, "message": "向量库信息已更新"})
|
||||
return jsonify({"error": "更新失败"}), 500
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>', methods=['DELETE'])
|
||||
@require_gateway_auth
|
||||
def delete_collection(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
删除向量库
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
查询参数:
|
||||
delete_documents: 是否删除文档源文件（默认 false）
|
||||
|
||||
Returns:
|
||||
成功: {"success": true, "message": "...", "deleted_documents": bool}
|
||||
失败: {"error": "..."} (400)
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
# 获取参数
|
||||
delete_documents = request.args.get('delete_documents', 'false').lower() == 'true'
|
||||
|
||||
success, message = kb_manager.delete_collection(kb_name, delete_documents)
|
||||
|
||||
if success:
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"message": message,
|
||||
"deleted_documents": delete_documents
|
||||
})
|
||||
return jsonify({"error": message}), 400
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/documents', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def list_collection_documents(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
获取向量库中的文档列表
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
Returns:
|
||||
{"collection": "...", "documents": [...], "total": N}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
documents = kb_manager.list_documents(kb_name)
|
||||
|
||||
return jsonify({
|
||||
"collection": kb_name,
|
||||
"documents": documents,
|
||||
"total": len(documents)
|
||||
})
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/chunks', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def list_collection_chunks(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
获取向量库中的切片列表
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
查询参数:
|
||||
document_id: 过滤指定文档的切片（可选）
|
||||
limit: 返回数量限制（默认 100）
|
||||
offset: 偏移量（默认 0）
|
||||
|
||||
Returns:
|
||||
{"collection": "...", "chunks": [...], "total": N}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
# 可选过滤参数
|
||||
document_id = request.args.get('document_id')
|
||||
limit = request.args.get('limit', 100, type=int)
|
||||
offset = request.args.get('offset', 0, type=int)
|
||||
|
||||
chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset)
|
||||
|
||||
return jsonify({
|
||||
"collection": kb_name,
|
||||
"chunks": chunks,
|
||||
"total": len(chunks)
|
||||
})
|
||||
|
||||
|
||||
@kb_bp.route('/documents/sync', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def sync_documents() -> Tuple[Any, int]:
|
||||
"""
|
||||
触发文档向量化同步
|
||||
|
||||
扫描文档目录，检测新增、修改、删除的文件，
|
||||
自动更新向量库索引。
|
||||
|
||||
请求体:
|
||||
{
|
||||
"collection": "向量库名称" // 可选，不传则同步所有
|
||||
}
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"results": [{"collection": "...", "status": "...", ...}],
|
||||
"synced_count": N
|
||||
}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
user = request.current_user
|
||||
data = request.json or {}
|
||||
target_collection = data.get('collection')
|
||||
|
||||
# 确定要同步的向量库
|
||||
if target_collection:
|
||||
collections_to_sync = [target_collection]
|
||||
else:
|
||||
# 同步所有向量库
|
||||
all_collections = kb_manager.list_collections()
|
||||
collections_to_sync = [c.name for c in all_collections]
|
||||
|
||||
if not collections_to_sync:
|
||||
return jsonify({"error": "没有可同步的向量库"}), 400
|
||||
|
||||
# 执行同步
|
||||
results = []
|
||||
|
||||
# 使用 sync_service 执行同步
|
||||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||||
|
||||
if sync_service:
|
||||
try:
|
||||
sync_result = sync_service.sync_now()
|
||||
results.append({
|
||||
"collection": "all",
|
||||
"status": "success",
|
||||
"message": f"同步完成: 处理 {sync_result.documents_processed} 个文档",
|
||||
"details": {
|
||||
"added": sync_result.documents_added,
|
||||
"modified": sync_result.documents_modified,
|
||||
"deleted": sync_result.documents_deleted,
|
||||
"errors": sync_result.errors
|
||||
}
|
||||
})
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"collection": "all",
|
||||
"status": "error",
|
||||
"message": str(e)
|
||||
})
|
||||
else:
|
||||
# 没有 sync_service，返回提示
|
||||
for coll_name in collections_to_sync:
|
||||
results.append({
|
||||
"collection": coll_name,
|
||||
"status": "warning",
|
||||
"message": "同步服务不可用，请使用 POST /sync 端点"
|
||||
})
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"results": results,
|
||||
"synced_count": len([r for r in results if r["status"] == "success"])
|
||||
})
|
||||
|
||||
|
||||
@kb_bp.route('/debug/scan', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def debug_scan() -> Tuple[Any, int]:
|
||||
"""
|
||||
调试：查看同步服务扫描的文件
|
||||
|
||||
返回文档目录的文件列表和同步服务状态，
|
||||
用于排查文档同步问题。
|
||||
|
||||
Returns:
|
||||
{
|
||||
"documents_path": "...",
|
||||
"exists": bool,
|
||||
"files": [...],
|
||||
"sync_service_path": "...",
|
||||
"scanned_count": N
|
||||
}
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
import os
|
||||
|
||||
result = {
|
||||
"documents_path": DOCUMENTS_PATH,
|
||||
"exists": os.path.exists(DOCUMENTS_PATH),
|
||||
"files": []
|
||||
}
|
||||
|
||||
if os.path.exists(DOCUMENTS_PATH):
|
||||
for root, dirs, files in os.walk(DOCUMENTS_PATH):
|
||||
for f in files:
|
||||
fp = os.path.join(root, f)
|
||||
rel = os.path.relpath(fp, DOCUMENTS_PATH)
|
||||
result["files"].append({
|
||||
"rel_path": rel,
|
||||
"size": os.path.getsize(fp),
|
||||
"ext": os.path.splitext(f)[1].lower()
|
||||
})
|
||||
|
||||
# 也检查同步服务的路径
|
||||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||||
if sync_service:
|
||||
result["sync_service_path"] = sync_service.documents_path
|
||||
result["sync_service_path_exists"] = os.path.exists(sync_service.documents_path)
|
||||
|
||||
# 直接调用 scan_documents
|
||||
try:
|
||||
scanned = sync_service.scan_documents()
|
||||
result["scanned_count"] = len(scanned)
|
||||
result["scanned_ids"] = list(scanned.keys())[:10]
|
||||
except Exception as e:
|
||||
result["scan_error"] = str(e)
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/reindex', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def reindex_collection(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
强制重新向量化指定集合的所有文档
|
||||
|
||||
清除该集合的文档哈希记录，触发完整重新索引。
|
||||
适用于文档内容更新后需要重建索引的场景。
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"message": "...",
|
||||
"documents_processed": N,
|
||||
"documents_added": N,
|
||||
"errors": [...]
|
||||
}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
# 清除该集合的文档哈希记录
|
||||
try:
|
||||
from data.db import get_connection
|
||||
with get_connection("knowledge") as conn:
|
||||
cursor = conn.cursor()
|
||||
# 删除以 "{kb_name}/" 或 "{kb_name}\" 开头的文档哈希（兼容 Windows 和 Linux）
|
||||
cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? OR document_id LIKE ?",
|
||||
(f"{kb_name}/%", f"{kb_name}\\%"))
|
||||
deleted = cursor.rowcount
|
||||
logger.info(f"已清除 {deleted} 条哈希记录: {kb_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清除哈希记录失败: {e}")
|
||||
|
||||
# 触发同步
|
||||
sync_service = current_app.config.get('SYNC_SERVICE')
|
||||
if sync_service:
|
||||
try:
|
||||
result = sync_service.sync_now()
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"message": f"重新索引完成: 处理 {result.documents_processed} 个文档",
|
||||
"documents_processed": result.documents_processed,
|
||||
"documents_added": result.documents_added,
|
||||
"errors": result.errors
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
else:
|
||||
return jsonify({"error": "同步服务不可用"}), 503
|
||||
|
||||
|
||||
@kb_bp.route('/kb/route', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def test_routing() -> Tuple[Any, int]:
|
||||
"""
|
||||
测试知识库路由（调试用）
|
||||
|
||||
分析查询意图，返回目标向量库列表和意图分析结果。
|
||||
|
||||
请求体:
|
||||
{"query": "用户查询"}
|
||||
|
||||
Returns:
|
||||
{
|
||||
"query": "...",
|
||||
"user_role": "...",
|
||||
"user_department": "...",
|
||||
"target_collections": [...],
|
||||
"intent": {
|
||||
"is_general": bool,
|
||||
"department": "...",
|
||||
"confidence": float,
|
||||
"keywords": [...],
|
||||
"reason": "..."
|
||||
}
|
||||
}
|
||||
"""
|
||||
_, kb_router, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
from knowledge.router import route_query
|
||||
|
||||
user = request.current_user
|
||||
data = request.json or {}
|
||||
query = data.get('query', '')
|
||||
|
||||
if not query:
|
||||
return jsonify({"error": "请提供查询内容"}), 400
|
||||
|
||||
# 获取路由结果
|
||||
target_kbs = route_query(
|
||||
query,
|
||||
user.get("role", "user"),
|
||||
user.get("department", "")
|
||||
)
|
||||
|
||||
# 获取意图分析
|
||||
intent = kb_router.analyze_intent(query)
|
||||
|
||||
return jsonify({
|
||||
"query": query,
|
||||
"user_role": user.get("role"),
|
||||
"user_department": user.get("department", ""),
|
||||
"target_collections": target_kbs,
|
||||
"intent": {
|
||||
"is_general": intent.is_general,
|
||||
"department": intent.department,
|
||||
"confidence": intent.confidence,
|
||||
"keywords": intent.keywords,
|
||||
"reason": intent.reason
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
# ==================== 文档版本管理 API ====================
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/deprecate', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def deprecate_document(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
废止文档（软删除）
|
||||
|
||||
将文档标记为已废止，相关切片在检索时被过滤。
|
||||
文档数据保留，可通过 restore_document 恢复。
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
filename: 文档文件名
|
||||
|
||||
请求体:
|
||||
{"reason": "废止原因"}
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"deprecated_chunks": N,
|
||||
"document_id": "...",
|
||||
"collection": "...",
|
||||
"deprecated_date": "ISO 8601 时间"
|
||||
}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
user = request.current_user
|
||||
data = request.json or {}
|
||||
reason = data.get('reason', '文档已废止')
|
||||
|
||||
try:
|
||||
result = kb_manager.deprecate_document(
|
||||
kb_name,
|
||||
filename,
|
||||
reason,
|
||||
deprecated_by=user.get('user_id', 'unknown')
|
||||
)
|
||||
return jsonify(result)
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/restore', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def restore_document(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
恢复已废止的文档
|
||||
|
||||
将已废止的文档恢复为有效状态，相关切片重新参与检索。
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
filename: 文档文件名
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"restored_chunks": N,
|
||||
"document_id": "...",
|
||||
"collection": "..."
|
||||
}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
try:
|
||||
result = kb_manager.restore_document(kb_name, filename)
|
||||
return jsonify(result)
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/versions', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
获取文档版本历史
|
||||
|
||||
返回文档的所有版本记录，包括状态变更、时间戳等信息。
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
filename: 文档文件名
|
||||
|
||||
查询参数:
|
||||
limit: 返回数量限制（默认 10）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"document_id": "...",
|
||||
"collection": "...",
|
||||
"versions": [
|
||||
{
|
||||
"version": "v2",
|
||||
"status": "active",
|
||||
"created_at": "ISO 8601",
|
||||
"chunk_count": N
|
||||
},
|
||||
...
|
||||
],
|
||||
"total": N
|
||||
}
|
||||
"""
|
||||
_, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
limit = request.args.get('limit', 10, type=int)
|
||||
|
||||
try:
|
||||
from knowledge.document_versions import get_version_query
|
||||
version_query = get_version_query()
|
||||
|
||||
versions = version_query.get_document_history(kb_name, filename, limit)
|
||||
versions_data = [v.to_dict() for v in versions]
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"document_id": filename,
|
||||
"collection": kb_name,
|
||||
"versions": versions_data,
|
||||
"total": len(versions_data)
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@kb_bp.route('/collections/<kb_name>/update-image-descriptions', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def update_image_descriptions(kb_name: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
更新图片切片的轻量级描述
|
||||
|
||||
重新提取已入库文档中图片切片的图号/表号信息，
|
||||
生成新的轻量级描述，提高检索准确度。
|
||||
|
||||
适用场景:
|
||||
- 文档入库时未提取图号
|
||||
- 图号提取规则更新后需要重新处理
|
||||
|
||||
Args:
|
||||
kb_name: 向量库名称
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"image_count": N,
|
||||
"updated_count": M
|
||||
}
|
||||
"""
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
try:
|
||||
result = kb_manager.update_image_descriptions(kb_name)
|
||||
return jsonify(result)
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@kb_bp.route('/collections/sync-vlm-cache', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def sync_vlm_cache() -> Tuple[Any, int]:
|
||||
"""
|
||||
同步 VLM 缓存到向量库元数据
|
||||
|
||||
将 .data/cache/vlm/ 中已生成的图片描述同步到向量库的 vlm_desc 字段。
|
||||
同步后，检索时可直接从向量库获取 VLM 描述，无需读取文件系统。
|
||||
|
||||
请求参数:
|
||||
- collection: 可选，指定同步的集合名，不传则同步所有集合
|
||||
- dry_run: 可选，设为 true 只预览不执行
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"total_cache_files": N,
|
||||
"synced_count": M,
|
||||
"skipped_count": K,
|
||||
"details": [...]
|
||||
}
|
||||
"""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json as json_module
|
||||
|
||||
kb_manager, _, err = _require_multi_kb()
|
||||
if err:
|
||||
return err
|
||||
|
||||
data = request.get_json() or {}
|
||||
collection_filter = data.get('collection')
|
||||
dry_run = data.get('dry_run', False)
|
||||
|
||||
vlm_cache_dir = Path(".data/cache/vlm")
|
||||
images_dir = Path(".data/images")
|
||||
|
||||
if not vlm_cache_dir.exists():
|
||||
return jsonify({"success": False, "error": "VLM 缓存目录不存在"}), 400
|
||||
|
||||
# 获取所有 VLM 缓存文件
|
||||
cache_files = list(vlm_cache_dir.glob("*.txt"))
|
||||
if not cache_files:
|
||||
return jsonify({"success": True, "total_cache_files": 0, "synced_count": 0, "message": "无 VLM 缓存文件"})
|
||||
|
||||
# 构建图片 MD5 → 文件名 的映射
|
||||
image_hash_map = {}
|
||||
if images_dir.exists():
|
||||
for img_file in images_dir.glob("*.png"):
|
||||
img_hash = hashlib.md5(img_file.read_bytes()).hexdigest()
|
||||
image_hash_map[img_hash] = img_file.name
|
||||
|
||||
# 结果统计
|
||||
synced_count = 0
|
||||
skipped_count = 0
|
||||
details = []
|
||||
|
||||
# 获取所有集合
|
||||
collections = kb_manager.list_collections()
|
||||
if collection_filter:
|
||||
collections = [c for c in collections if c.name == collection_filter]
|
||||
|
||||
for cache_file in cache_files:
|
||||
cache_hash = cache_file.stem # 文件名即 MD5
|
||||
|
||||
# 查找对应的图片文件名
|
||||
image_filename = image_hash_map.get(cache_hash)
|
||||
if not image_filename:
|
||||
skipped_count += 1
|
||||
details.append({"cache": cache_file.name, "status": "skipped", "reason": "图片文件不存在"})
|
||||
continue
|
||||
|
||||
# 读取 VLM 描述
|
||||
vlm_desc = cache_file.read_text(encoding='utf-8')
|
||||
|
||||
# 在所有集合中查找该图片切片
|
||||
found = False
|
||||
for coll in collections:
|
||||
collection = kb_manager.get_collection(coll.name)
|
||||
if not collection:
|
||||
continue
|
||||
|
||||
# 查找 image_path 匹配的切片
|
||||
result = collection.get(where={"image_path": image_filename})
|
||||
|
||||
if result['ids']:
|
||||
found = True
|
||||
if not dry_run:
|
||||
# 更新元数据
|
||||
updated_metadatas = []
|
||||
for meta in result['metadatas']:
|
||||
meta['vlm_desc'] = vlm_desc
|
||||
meta['has_vlm_desc'] = True
|
||||
updated_metadatas.append(meta)
|
||||
|
||||
collection.update(
|
||||
ids=result['ids'],
|
||||
metadatas=updated_metadatas
|
||||
)
|
||||
|
||||
synced_count += len(result['ids'])
|
||||
details.append({
|
||||
"cache": cache_file.name,
|
||||
"image": image_filename,
|
||||
"collection": coll.name,
|
||||
"count": len(result['ids']),
|
||||
"status": "synced" if not dry_run else "preview"
|
||||
})
|
||||
|
||||
if not found:
|
||||
skipped_count += 1
|
||||
details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"})
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"total_cache_files": len(cache_files),
|
||||
"synced_count": synced_count,
|
||||
"skipped_count": skipped_count,
|
||||
"dry_run": dry_run,
|
||||
"details": details[:20] # 只返回前 20 条详情
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user