jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(api): 统一响应格式迁移 + 异步任务系统 + 状态码体系完善

Browse Source 
- 将全部路由文件（12个）的 jsonify 响应迁移至 success_response/error_response 统一格式
- 修复 sync_routes.py error_response 参数错误（P0）
- 新增异步任务系统：task_registry + task_routes
- 新增状态码：TASK_NOT_FOUND(4014)、TASK_CONFLICT(4015)、REINDEX_ERROR(5040)
- 修正 task_routes/exam_pkg 中语义不匹配的状态码
- 更新 curl 测试手册、后端对接规范文档
- 添加缓存性能报告和 Redis 迁移计划
This commit is contained in:
lacerate551
2026-06-05 22:56:00 +08:00
parent 54a6815ad4
commit 183a57e7f1
19 changed files with 2659 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
api/kb_routes.py
View File

@@ -37,6 +37,13 @@ from typing import Tuple, Optional, Any
from flask import Blueprint, request, jsonify, current_app
import logging
from core.status_codes import (
SUCCESS, CREATED, DELETE_SUCCESS, UPDATE_SUCCESS, SYNC_SUCCESS,
BAD_REQUEST, NOT_FOUND, COLLECTION_NOT_FOUND, NO_COLLECTION, TASK_CONFLICT,
INTERNAL_ERROR, SERVICE_UNAVAILABLE, SYNC_ERROR, REINDEX_ERROR
)
from api.response_utils import success_response, error_response
logger = logging.getLogger(__name__)
from auth.gateway import require_gateway_auth
@@ -133,10 +140,7 @@ def list_collections() -> Tuple[Any, int]:
"description": coll.description
})
return jsonify({
"collections": result,
"total": len(result)
})
return success_response(data={"collections": result, "total": len(result)})
@kb_bp.route('/collections', methods=['POST'])
@@ -172,22 +176,19 @@ def create_collection() -> Tuple[Any, int]:
description = data.get('description', '')
if not name:
return jsonify({"error": "向量库名称不能为空"}), 400
return error_response("INVALID_NAME", BAD_REQUEST, "向量库名称不能为空", http_status=400)
# 验证名称格式（ChromaDB 限制）
if not name.replace('_', '').replace('-', '').isalnum():
return jsonify({
"error": "名称格式错误",
"message": "向量库名称只能包含字母、数字、下划线和连字符"
}), 400
return error_response("INVALID_NAME_FORMAT", BAD_REQUEST, "向量库名称只能包含字母、数字、下划线和连字符", http_status=400)
success, message = kb_manager.create_collection(
name, display_name, department, description
)
if success:
return jsonify({"success": True, "message": message, "name": name}), 201
return jsonify({"error": message}), 400
return success_response(data={"name": name}, status_code=CREATED, message=message, http_status=201)
return error_response("CREATE_FAILED", BAD_REQUEST, message, http_status=400)
@kb_bp.route('/collections/<kb_name>', methods=['PUT'])
@@ -220,7 +221,7 @@ def update_collection(kb_name: str) -> Tuple[Any, int]:
# 检查向量库是否存在
collections = kb_manager.list_collections()
if not any(c.name == kb_name for c in collections):
return jsonify({"error": f"向量库 '{kb_name}' 不存在"}), 404
return error_response("COLLECTION_NOT_FOUND", COLLECTION_NOT_FOUND, f"向量库 '{kb_name}' 不存在", http_status=404)
# 更新元数据
success = kb_manager.update_collection_metadata(
@@ -230,8 +231,8 @@ def update_collection(kb_name: str) -> Tuple[Any, int]:
)
if success:
return jsonify({"success": True, "message": "向量库信息已更新"})
return jsonify({"error": "更新失败"}), 500
return success_response(data=None, status_code=UPDATE_SUCCESS, message="向量库信息已更新")
return error_response("UPDATE_FAILED", INTERNAL_ERROR, "更新失败", http_status=500)
@kb_bp.route('/collections/<kb_name>', methods=['DELETE'])
@@ -260,12 +261,8 @@ def delete_collection(kb_name: str) -> Tuple[Any, int]:
success, message = kb_manager.delete_collection(kb_name, delete_documents)
if success:
return jsonify({
"success": True,
"message": message,
"deleted_documents": delete_documents
})
return jsonify({"error": message}), 400
return success_response(data={"deleted_documents": delete_documents}, status_code=DELETE_SUCCESS, message=message)
return error_response("DELETE_FAILED", BAD_REQUEST, message, http_status=400)
@kb_bp.route('/collections/<kb_name>/documents', methods=['GET'])
@@ -286,11 +283,7 @@ def list_collection_documents(kb_name: str) -> Tuple[Any, int]:
documents = kb_manager.list_documents(kb_name)
return jsonify({
"collection": kb_name,
"documents": documents,
"total": len(documents)
})
return success_response(data={"collection": kb_name, "documents": documents, "total": len(documents)})
@kb_bp.route('/collections/<kb_name>/chunks', methods=['GET'])
@@ -321,21 +314,16 @@ def list_collection_chunks(kb_name: str) -> Tuple[Any, int]:
chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset)
return jsonify({
"collection": kb_name,
"chunks": chunks,
"total": len(chunks)
})
return success_response(data={"collection": kb_name, "chunks": chunks, "total": len(chunks)})
@kb_bp.route('/documents/sync', methods=['POST'])
@require_gateway_auth
def sync_documents() -> Tuple[Any, int]:
"""
触发文档向量化同步
触发文档向量化同步（异步任务）
扫描文档目录，检测新增、修改、删除的文件，
自动更新向量库索引。
立即返回 task_id，后台线程执行同步。
请求体:
{
@@ -343,11 +331,7 @@ def sync_documents() -> Tuple[Any, int]:
}
Returns:
{
"success": true,
"results": [{"collection": "...", "status": "...", ...}],
"synced_count": N
}
{"success": true, "data": {"task_id": "xxx", "message": "..."}}
"""
kb_manager, _, err = _require_multi_kb()
if err:
@@ -355,7 +339,6 @@ def sync_documents() -> Tuple[Any, int]:
from config import DOCUMENTS_PATH
user = request.current_user
data = request.json or {}
target_collection = data.get('collection')
@@ -363,54 +346,67 @@ def sync_documents() -> Tuple[Any, int]:
if target_collection:
collections_to_sync = [target_collection]
else:
# 同步所有向量库
all_collections = kb_manager.list_collections()
collections_to_sync = [c.name for c in all_collections]
if not collections_to_sync:
return jsonify({"error": "没有可同步的向量库"}), 400
return error_response("NO_COLLECTION", NO_COLLECTION, "没有可同步的向量库", http_status=400)
# 执行同步
results = []
# 使用 sync_service 执行同步
sync_service = current_app.config.get('SYNC_SERVICE')
if not sync_service:
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503,
results=[{"collection": c, "status": "warning", "message": "同步服务不可用"} for c in collections_to_sync],
synced_count=0
)
if sync_service:
from core.task_registry import get_registry
registry = get_registry()
# 检查是否有正在运行的同步任务
running = registry.list_tasks(status='running', task_type='sync', limit=1)
if running:
return error_response("TASK_RUNNING", TASK_CONFLICT, f"同步任务正在执行中 (task_id: {running[0].id})", http_status=409)
desc = f"文档同步: {target_collection or '所有向量库'}"
task = registry.create_task('sync', desc)
def _do_sync(task, sync_svc):
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id, current=processed[0],
stage='处理文件',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_callback = sync_svc.on_change_callback
sync_svc.on_change_callback = on_change
try:
sync_result = sync_service.sync_now()
results.append({
"collection": "all",
"status": "success",
"message": f"同步完成: 处理 {sync_result.documents_processed} 个文档",
"details": {
"added": sync_result.documents_added,
"modified": sync_result.documents_modified,
"deleted": sync_result.documents_deleted,
"errors": sync_result.errors
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
sync_result = sync_svc.sync_now()
return {
'collection': target_collection or 'all',
'status': 'success',
'message': f"同步完成: 处理 {sync_result.documents_processed} 个文档",
'details': {
'added': sync_result.documents_added,
'modified': sync_result.documents_modified,
'deleted': sync_result.documents_deleted,
'errors': sync_result.errors,
}
})
except Exception as e:
logger.error(f"知识库操作异常: {e}")
results.append({
"collection": "all",
"status": "error",
"message": "操作失败"
})
else:
# 没有 sync_service，返回提示
for coll_name in collections_to_sync:
results.append({
"collection": coll_name,
"status": "warning",
"message": "同步服务不可用，请使用 POST /sync 端点"
})
}
finally:
sync_svc.on_change_callback = old_callback
return jsonify({
"success": True,
"results": results,
"synced_count": len([r for r in results if r["status"] == "success"])
})
registry.start_task(task.id, _do_sync, sync_service)
return success_response(
data={"task_id": task.id, "message": f"同步任务已启动，通过 GET /tasks/{task.id} 查询进度"},
status_code=SYNC_SUCCESS,
message="同步任务已启动"
)
@kb_bp.route('/debug/scan', methods=['GET'])
@@ -473,22 +469,16 @@ def debug_scan() -> Tuple[Any, int]:
@require_gateway_auth
def reindex_collection(kb_name: str) -> Tuple[Any, int]:
"""
强制重新向量化指定集合的所有文档
强制重新向量化指定集合的所有文档（异步任务）
清除该集合的文档哈希记录，触发完整重新索引。
适用于文档内容更新后需要重建索引的场景。
立即返回 task_id，后台线程执行重建。
Args:
kb_name: 向量库名称
Returns:
{
"success": true,
"message": "...",
"documents_processed": N,
"documents_added": N,
"errors": [...]
}
{"success": true, "data": {"task_id": "xxx", "message": "..."}}
"""
kb_manager, _, err = _require_multi_kb()
if err:
@@ -496,14 +486,12 @@ def reindex_collection(kb_name: str) -> Tuple[Any, int]:
from config import DOCUMENTS_PATH
# 清除该集合的文档哈希记录
# 清除该集合的文档哈希记录（同步完成，很快）
try:
from data.db import get_connection
with get_connection("knowledge") as conn:
cursor = conn.cursor()
# 转义 LIKE 通配符，防止 kb_name 中的 % 或 _ 导致非预期匹配
escaped_kb = kb_name.replace('%', '\\%').replace('_', '\\_')
# 删除以 "{kb_name}/" 或 "{kb_name}\" 开头的文档哈希（兼容 Windows 和 Linux）
cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? ESCAPE '\\' OR document_id LIKE ? ESCAPE '\\'",
(f"{escaped_kb}/%", f"{escaped_kb}\\%"))
deleted = cursor.rowcount
@@ -511,23 +499,54 @@ def reindex_collection(kb_name: str) -> Tuple[Any, int]:
except Exception as e:
logger.warning(f"清除哈希记录失败: {e}")
# 触发同步
sync_service = current_app.config.get('SYNC_SERVICE')
if sync_service:
if not sync_service:
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503)
from core.task_registry import get_registry
registry = get_registry()
# 检查是否有正在运行的重建任务
running = registry.list_tasks(status='running', task_type='reindex', limit=1)
if running:
return error_response("TASK_RUNNING", TASK_CONFLICT, f"重建任务正在执行中 (task_id: {running[0].id})", http_status=409)
task = registry.create_task('reindex', f'重建索引: {kb_name}')
def _do_reindex(task, sync_svc, kb):
"""后台执行重建索引"""
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id,
current=processed[0],
stage='重新索引',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_callback = sync_svc.on_change_callback
sync_svc.on_change_callback = on_change
try:
result = sync_service.sync_now()
return jsonify({
"success": True,
"message": f"重新索引完成: 处理 {result.documents_processed} 个文档",
"documents_processed": result.documents_processed,
"documents_added": result.documents_added,
"errors": result.errors
})
except Exception as e:
logger.error(f"reindex 异常: {e}")
return jsonify({"error": "操作失败，请稍后重试"}), 500
else:
return jsonify({"error": "同步服务不可用"}), 503
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
result = sync_svc.sync_now()
return {
'message': f"重新索引完成: 处理 {result.documents_processed} 个文档",
'documents_processed': result.documents_processed,
'documents_added': result.documents_added,
'errors': result.errors,
}
finally:
sync_svc.on_change_callback = old_callback
registry.start_task(task.id, _do_reindex, sync_service, kb_name)
return success_response(
data={"task_id": task.id, "message": f"重建索引任务已启动: {kb_name}，通过 GET /tasks/{task.id} 查询进度"},
status_code=SYNC_SUCCESS,
message="重建索引任务已启动"
)
@kb_bp.route('/kb/route', methods=['POST'])
@@ -567,7 +586,7 @@ def test_routing() -> Tuple[Any, int]:
query = data.get('query', '')
if not query:
return jsonify({"error": "请提供查询内容"}), 400
return error_response("MISSING_PARAMS", BAD_REQUEST, "请提供查询内容", http_status=400)
# 获取路由结果
target_kbs = route_query(
@@ -579,7 +598,7 @@ def test_routing() -> Tuple[Any, int]:
# 获取意图分析
intent = kb_router.analyze_intent(query)
return jsonify({
return success_response(data={
"query": query,
"user_role": user.get("role"),
"user_department": user.get("department", ""),
@@ -636,10 +655,10 @@ def deprecate_document(kb_name: str, filename: str) -> Tuple[Any, int]:
reason,
deprecated_by=user.get('user_id', 'unknown')
)
return jsonify(result)
return success_response(data=result)
except Exception as e:
logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/restore', methods=['POST'])
@@ -668,10 +687,10 @@ def restore_document(kb_name: str, filename: str) -> Tuple[Any, int]:
try:
result = kb_manager.restore_document(kb_name, filename)
return jsonify(result)
return success_response(data=result)
except Exception as e:
logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/versions', methods=['GET'])
@@ -719,8 +738,7 @@ def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
versions = version_query.get_document_history(kb_name, filename, limit)
versions_data = [v.to_dict() for v in versions]
return jsonify({
"success": True,
return success_response(data={
"document_id": filename,
"collection": kb_name,
"versions": versions_data,
@@ -728,7 +746,7 @@ def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
})
except Exception as e:
logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/update-image-descriptions', methods=['POST'])
@@ -760,10 +778,10 @@ def update_image_descriptions(kb_name: str) -> Tuple[Any, int]:
try:
result = kb_manager.update_image_descriptions(kb_name)
return jsonify(result)
return success_response(data=result)
except Exception as e:
logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败，请稍后重试"}), 500
return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)
@kb_bp.route('/collections/sync-vlm-cache', methods=['POST'])
@@ -804,12 +822,12 @@ def sync_vlm_cache() -> Tuple[Any, int]:
images_dir = Path(".data/images")
if not vlm_cache_dir.exists():
return jsonify({"success": False, "error": "VLM 缓存目录不存在"}), 400
return error_response("VLM_CACHE_NOT_FOUND", BAD_REQUEST, "VLM 缓存目录不存在", http_status=400)
# 获取所有 VLM 缓存文件
cache_files = list(vlm_cache_dir.glob("*.txt"))
if not cache_files:
return jsonify({"success": True, "total_cache_files": 0, "synced_count": 0, "message": "无 VLM 缓存文件"})
return success_response(data={"total_cache_files": 0, "synced_count": 0}, message="无 VLM 缓存文件")
# 构建图片 MD5 → 文件名 的映射
image_hash_map = {}
@@ -879,8 +897,7 @@ def sync_vlm_cache() -> Tuple[Any, int]:
skipped_count += 1
details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"})
return jsonify({
"success": True,
return success_response(data={
"total_cache_files": len(cache_files),
"synced_count": synced_count,
"skipped_count": skipped_count,