jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(api): 统一响应格式迁移 + 异步任务系统 + 状态码体系完善

Browse Source 
- 将全部路由文件（12个）的 jsonify 响应迁移至 success_response/error_response 统一格式
- 修复 sync_routes.py error_response 参数错误（P0）
- 新增异步任务系统：task_registry + task_routes
- 新增状态码：TASK_NOT_FOUND(4014)、TASK_CONFLICT(4015)、REINDEX_ERROR(5040)
- 修正 task_routes/exam_pkg 中语义不匹配的状态码
- 更新 curl 测试手册、后端对接规范文档
- 添加缓存性能报告和 Redis 迁移计划
This commit is contained in:
lacerate551
2026-06-05 22:56:00 +08:00
parent 54a6815ad4
commit 183a57e7f1
19 changed files with 2659 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
api/document_routes.py
View File

@@ -39,16 +39,18 @@ import re
import uuid
from datetime import datetime
from typing import Optional, Tuple, Any, List, Dict
from flask import Blueprint, request, jsonify
from flask import Blueprint, request
import logging
logger = logging.getLogger(__name__)
from werkzeug.utils import secure_filename
from auth.gateway import require_gateway_auth
from core.status_codes import (
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST,
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION,
FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST, SUCCESS,
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION, NOT_FOUND,
FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR,
SERVICE_UNAVAILABLE, DELETE_SUCCESS, UPDATE_SUCCESS, NO_CONTENT,
FORBIDDEN
)
from api.response_utils import success_response, error_response
@@ -172,7 +174,7 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
仅在 DEV_MODE=true 时可用
"""
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
return jsonify({"error": "仅开发环境可用"}), 403
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
from config import DOCUMENTS_PATH
from flask import send_from_directory
@@ -181,9 +183,9 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError:
return jsonify({"error": "非法路径"}), 403
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
directory = os.path.dirname(filepath)
filename = os.path.basename(filepath)
@@ -334,10 +336,24 @@ def upload_document() -> Tuple[Any, int]:
except Exception as e:
logger.warning(f"标记旧版本失败: {e}")
# 6. 触发向量化
# 6. 触发向量化（异步任务）
sync_status = "已保存，等待手动同步"
sync_service = _get_sync_service()
task_id = None
if sync_service:
from core.task_registry import get_registry
registry = get_registry()
task = registry.create_task('upload', f"向量化: {filename}")
task_id = task.id
def _do_vectorize(task, svc, change_obj, fname):
"""后台执行向量化"""
registry.update_progress(task.id, stage='向量化', message=f"正在处理: {fname}")
svc.process_change(change_obj)
return {'filename': fname, 'sync_status': '已添加到向量库'}
try:
from knowledge.sync import DocumentChange, ChangeType
change = DocumentChange(
@@ -348,11 +364,12 @@ def upload_document() -> Tuple[Any, int]:
new_hash=sync_service.calculate_file_hash(filepath),
change_time=datetime.now()
)
sync_service.process_change(change)
sync_status = "已保存并添加到向量库"
registry.start_task(task.id, _do_vectorize, sync_service, change, filename)
sync_status = "已保存，向量化任务已启动"
except Exception as e:
logger.warning(f"向量化失败: {e}")
sync_status = "已保存，向量化失败"
logger.warning(f"创建向量化任务失败: {e}")
registry.fail_task(task.id, str(e))
sync_status = "已保存，向量化任务创建失败"
return success_response(
data={
@@ -363,7 +380,8 @@ def upload_document() -> Tuple[Any, int]:
"size": file_size,
"replaced": replaced
},
"sync_status": sync_status
"sync_status": sync_status,
"task_id": task_id
},
status_code=UPLOAD_SUCCESS,
message=f"文件上传成功，{sync_status}"
@@ -506,14 +524,55 @@ def batch_upload_documents() -> Tuple[Any, int]:
"message": "上传处理失败"
})
success_count = len([r for r in results if r["status"] == "success"])
task_id = None
# 批量上传完成后自动触发向量化（异步任务）
sync_service = _get_sync_service()
if sync_service and success_count > 0:
from core.task_registry import get_registry
registry = get_registry()
running = registry.list_tasks(status='running', task_type='batch_upload', limit=1)
if not running:
task = registry.create_task('batch_upload', f"批量向量化: {success_count} 个文件", total=success_count)
task_id = task.id
def _do_batch_vectorize(task, svc, count):
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id, current=processed[0], total=count,
stage='批量向量化',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_cb = svc.on_change_callback
svc.on_change_callback = on_change
try:
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
result = svc.sync_now()
return {
'synced': result.documents_processed,
'added': result.documents_added,
'errors': result.errors,
}
finally:
svc.on_change_callback = old_cb
registry.start_task(task.id, _do_batch_vectorize, sync_service, success_count)
return success_response(
data={
"total": len(results),
"success_count": len([r for r in results if r["status"] == "success"]),
"results": results
"success_count": success_count,
"results": results,
"task_id": task_id
},
status_code=BATCH_UPLOAD_SUCCESS,
message=f"批量上传完成，成功 {len([r for r in results if r['status'] == 'success'])}/{len(results)} 个文件"
message=f"批量上传完成，成功 {success_count}/{len(results)} 个文件"
)
@@ -590,10 +649,7 @@ def list_documents() -> Tuple[Any, int]:
# 按修改时间倒序
documents.sort(key=lambda x: x['last_modified'], reverse=True)
return jsonify({
"documents": documents,
"total": len(documents)
})
return success_response(data={"documents": documents, "total": len(documents)})
@document_bp.route('/documents/<path:doc_path>/status', methods=['GET'])
@@ -617,12 +673,12 @@ def get_document_status(doc_path: str) -> Tuple[Any, int]:
"""
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径
parts = doc_path.split('/')
if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0]
filename = '/'.join(parts[1:])
@@ -638,16 +694,14 @@ def get_document_status(doc_path: str) -> Tuple[Any, int]:
if not doc_info:
if file_on_disk:
return jsonify({
"success": True,
return success_response(data={
"status": "unprocessed",
"chunk_count": 0,
"last_processed": None
})
return jsonify({"error": "文档不存在"}), 404
return error_response("NOT_FOUND", NOT_FOUND, "文档不存在", http_status=404)
return jsonify({
"success": True,
return success_response(data={
"status": doc_info.get("status", "unknown"),
"chunk_count": doc_info.get("total_chunks", 0),
"last_processed": doc_info.get("effective_date") or doc_info.get("version")
@@ -674,35 +728,35 @@ def update_document(doc_path: str) -> Tuple[Any, int]:
from config import DOCUMENTS_PATH
if 'file' not in request.files:
return jsonify({"error": "没有上传文件"}), 400
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
file = request.files['file']
if file.filename == '':
return jsonify({"error": "没有选择文件"}), 400
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
# 文件类型校验
ext = os.path.splitext(file.filename)[1].lower()
if ext not in ALLOWED_EXTENSIONS:
return jsonify({"error": f"不支持的文件类型: {ext}"}), 400
return error_response("UNSUPPORTED_FORMAT", UNSUPPORTED_FORMAT, f"不支持的文件类型: {ext}", http_status=400)
# 文件大小校验
file.seek(0, 2) # 跳到文件末尾获取大小
file_size = file.tell()
file.seek(0) # 回到文件开头
if file_size > MAX_FILE_SIZE:
return jsonify({"error": f"文件过大（最大 {MAX_FILE_SIZE // 1024 // 1024}MB）"}), 400
return error_response("FILE_TOO_LARGE", FILE_TOO_LARGE, f"文件过大（最大 {MAX_FILE_SIZE // 1024 // 1024}MB）", http_status=400)
# 解析路径
parts = doc_path.split('/')
if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError:
return jsonify({"error": "非法路径"}), 403
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
# 覆盖文件
file.save(filepath)
@@ -726,10 +780,7 @@ def update_document(doc_path: str) -> Tuple[Any, int]:
except Exception as e:
logger.warning(f"重新向量化失败: {e}")
return jsonify({
"success": True,
"message": "文件已更新"
})
return success_response(message="文件已更新")
@document_bp.route('/documents/<path:doc_path>', methods=['DELETE'])
@@ -754,7 +805,7 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
# 解析路径
parts = doc_path.split('/')
if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0]
filename = '/'.join(parts[1:])
@@ -765,9 +816,9 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError:
return jsonify({"error": "非法路径"}), 403
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
try:
# 1. 从向量库删除（source 存的是文件名，不是完整路径）
@@ -778,14 +829,11 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
# 2. 删除文件
os.remove(filepath)
return jsonify({
"success": True,
"message": "文档已删除"
})
return success_response(status_code=DELETE_SUCCESS, message="文档已删除")
except Exception as e:
logger.error(f"删除文档异常: {e}")
return jsonify({"error": "删除失败，请稍后重试"}), 500
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败，请稍后重试", http_status=500)
@document_bp.route('/documents/<path:doc_path>/chunks', methods=['GET'])
@@ -810,12 +858,12 @@ def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
"""
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径
parts = doc_path.split('/')
if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0]
# 目录名即向量库名
@@ -823,8 +871,7 @@ def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
chunks = kb_manager.get_document_chunks(collection, os.path.basename(doc_path))
return jsonify({
"success": True,
return success_response(data={
"document_id": doc_path,
"collection": collection,
"chunks": chunks,
@@ -860,12 +907,12 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
"""
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径
parts = doc_path.split('/')
if len(parts) < 2:
return jsonify({"error": "无效的文档路径，格式: collection/filename"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径，格式: collection/filename", http_status=400)
collection = parts[0]
filename = os.path.basename(doc_path)
@@ -880,7 +927,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
# 获取所有切片
all_chunks = kb_manager.get_document_chunks(collection, filename)
if not all_chunks:
return jsonify({"error": f"文档 '{filename}' 不存在或无切片"}), 404
return error_response("NOT_FOUND", NOT_FOUND, f"文档 '{filename}' 不存在或无切片", http_status=404)
total = len(all_chunks)
@@ -889,8 +936,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
preview_chunks = all_chunks[:5]
for c in preview_chunks:
c['is_target'] = False
return jsonify({
"success": True,
return success_response(data={
"collection": collection,
"source": filename,
"total_chunks": total,
@@ -902,7 +948,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
try:
target_chunk_index = int(chunk_index_str)
except ValueError:
return jsonify({"error": "chunk_index 必须为整数"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "chunk_index 必须为整数", http_status=400)
# 按 chunk_index 排序（Chroma 返回顺序不保证有序）
all_chunks.sort(key=lambda c: c.get('metadata', {}).get('chunk_index', 0))
@@ -923,9 +969,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
(c.get('metadata', {}).get('chunk_index', 0) for c in all_chunks),
default=total - 1
)
return jsonify({
"error": f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})"
}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})", http_status=400)
# 截取上下文窗口
start = max(0, target_pos - context_count)
@@ -936,8 +980,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
for i, c in enumerate(window):
c['is_target'] = (start + i == target_pos)
return jsonify({
"success": True,
return success_response(data={
"collection": collection,
"source": filename,
"total_chunks": total,
@@ -968,7 +1011,7 @@ def create_chunk() -> Tuple[Any, int]:
"""
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
data = request.json or {}
collection = data.get('collection')
@@ -976,17 +1019,13 @@ def create_chunk() -> Tuple[Any, int]:
metadata = data.get('metadata', {})
if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
if not content:
return jsonify({"error": "切片内容不能为空"}), 400
return error_response("NO_CONTENT", NO_CONTENT, "切片内容不能为空", http_status=400)
chunk_id = kb_manager.add_chunk(collection, content, metadata)
return jsonify({
"success": True,
"chunk_id": chunk_id,
"message": "切片已添加"
})
return success_response(data={"chunk_id": chunk_id}, message="切片已添加")
@document_bp.route('/chunks/<chunk_id>', methods=['PUT'])
@@ -1012,7 +1051,7 @@ def update_chunk(chunk_id: str) -> Tuple[Any, int]:
"""
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
data = request.json or {}
collection = data.get('collection')
@@ -1020,13 +1059,13 @@ def update_chunk(chunk_id: str) -> Tuple[Any, int]:
metadata = data.get('metadata')
if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
success = kb_manager.update_chunk(collection, chunk_id, content=content, metadata=metadata)
if success:
return jsonify({"success": True, "message": "切片已更新"})
return jsonify({"error": "更新失败"}), 500
return success_response(message="切片已更新")
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "更新失败", http_status=500)
@document_bp.route('/chunks/<chunk_id>', methods=['DELETE'])
@@ -1054,11 +1093,11 @@ def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
collection = request.args.get('collection')
if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 删除切片，返回 (success, source_file)
success, source_file = kb_manager.delete_chunk(collection, chunk_id)
@@ -1083,8 +1122,8 @@ def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
logger.warning(f"清理哈希记录失败: {e}")
if success:
return jsonify({"success": True, "message": "切片已删除"})
return jsonify({"error": "删除失败"}), 500
return success_response(status_code=DELETE_SUCCESS, message="切片已删除")
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败", http_status=500)
@document_bp.route('/chunks/batch', methods=['DELETE'])
@@ -1110,13 +1149,13 @@ def delete_chunks_by_source() -> Tuple[Any, int]:
source = data.get('source')
if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
if not source:
return jsonify({"error": "请指定文件名 (source)"}), 400
return error_response("BAD_REQUEST", BAD_REQUEST, "请指定文件名 (source)", http_status=400)
kb_manager = _get_kb_manager()
if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 批量删除该文件的所有切片
try:
@@ -1134,11 +1173,7 @@ def delete_chunks_by_source() -> Tuple[Any, int]:
except Exception as e:
logger.warning(f"清理哈希记录失败: {e}")
return jsonify({
"success": True,
"deleted_count": deleted_count,
"message": f"已删除 {deleted_count} 个切片"
})
return success_response(data={"deleted_count": deleted_count}, message=f"已删除 {deleted_count} 个切片")
except Exception as e:
logger.error(f"操作异常: {e}")
return jsonify({"error": "操作失败，请稍后重试"}), 500
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)