refactor(api): 统一响应格式迁移 + 异步任务系统 + 状态码体系完善

- 将全部路由文件(12个)的 jsonify 响应迁移至 success_response/error_response 统一格式
- 修复 sync_routes.py error_response 参数错误(P0)
- 新增异步任务系统:task_registry + task_routes
- 新增状态码:TASK_NOT_FOUND(4014)、TASK_CONFLICT(4015)、REINDEX_ERROR(5040)
- 修正 task_routes/exam_pkg 中语义不匹配的状态码
- 更新 curl 测试手册、后端对接规范文档
- 添加缓存性能报告和 Redis 迁移计划
This commit is contained in:
lacerate551
2026-06-05 22:56:00 +08:00
parent 54a6815ad4
commit 183a57e7f1
19 changed files with 2659 additions and 548 deletions

View File

@@ -132,6 +132,10 @@ def create_app() -> 'Flask':
from api.image_routes import image_bp from api.image_routes import image_bp
app.register_blueprint(image_bp) app.register_blueprint(image_bp)
# 异步任务查询
from api.task_routes import task_bp
app.register_blueprint(task_bp)
# 健康检查 # 健康检查
from api.auth_routes import auth_bp from api.auth_routes import auth_bp
app.register_blueprint(auth_bp) app.register_blueprint(auth_bp)
@@ -254,4 +258,5 @@ def _print_startup_info(app: 'Flask') -> None:
logger.info(" 切片管理: /chunks/*") logger.info(" 切片管理: /chunks/*")
logger.info(" 同步服务: /sync, /sync/status") logger.info(" 同步服务: /sync, /sync/status")
logger.info(" 图片服务: /images/*") logger.info(" 图片服务: /images/*")
logger.info(" 任务查询: /tasks, /tasks/<id>, /tasks/<id>/progress")
logger.info(" 健康检查: /health") logger.info(" 健康检查: /health")

View File

@@ -11,6 +11,8 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
from data.db import get_connection from data.db import get_connection
from core.status_codes import SUCCESS, INTERNAL_ERROR
from api.response_utils import success_response, error_response
audit_bp = Blueprint('audit', __name__) audit_bp = Blueprint('audit', __name__)
@@ -94,11 +96,11 @@ def get_audit_logs():
"timestamp": row[10] "timestamp": row[10]
}) })
return jsonify({"logs": logs, "total": total}) return success_response(data={"logs": logs, "total": total})
except Exception as e: except Exception as e:
logger.error(f"审计查询异常: {e}") logger.error(f"审计查询异常: {e}")
return jsonify({"error": "查询失败", "logs": [], "total": 0}), 500 return error_response("QUERY_FAILED", INTERNAL_ERROR, "查询失败", http_status=500)
def log_audit_event(user_id: str, username: str, action: str, def log_audit_event(user_id: str, username: str, action: str,

View File

@@ -10,6 +10,8 @@
from flask import Blueprint, request, jsonify from flask import Blueprint, request, jsonify
from auth.gateway import require_gateway_auth, require_role, get_user_permissions, MOCK_USERS from auth.gateway import require_gateway_auth, require_role, get_user_permissions, MOCK_USERS
from core.status_codes import SUCCESS, BAD_REQUEST, UNAUTHORIZED, FORBIDDEN, PERMISSION_DENIED, INTERNAL_ERROR
from api.response_utils import success_response, error_response
import os import os
from pathlib import Path from pathlib import Path
from dotenv import load_dotenv from dotenv import load_dotenv
@@ -52,7 +54,7 @@ def mock_login():
""" """
# 默认开启开发模式(生产环境需设置 DEV_MODE=false # 默认开启开发模式(生产环境需设置 DEV_MODE=false
if os.environ.get('DEV_MODE', 'true').lower() == 'false': if os.environ.get('DEV_MODE', 'true').lower() == 'false':
return jsonify({"error": "仅开发环境可用,请设置 DEV_MODE=true"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用,请设置 DEV_MODE=true", http_status=403)
data = request.json or {} data = request.json or {}
username = data.get('username') username = data.get('username')
@@ -60,9 +62,9 @@ def mock_login():
user = MOCK_USERS.get(username) user = MOCK_USERS.get(username)
if not user or user['password'] != password: if not user or user['password'] != password:
return jsonify({"error": "用户名或密码错误"}), 401 return error_response("UNAUTHORIZED", UNAUTHORIZED, "用户名或密码错误", http_status=401)
return jsonify({ return success_response(data={
"token": f"mock-token-{username}", "token": f"mock-token-{username}",
"user": { "user": {
"user_id": user['user_id'], "user_id": user['user_id'],
@@ -80,7 +82,7 @@ def get_stats():
"""获取系统统计信息(仅管理员)""" """获取系统统计信息(仅管理员)"""
from flask import current_app from flask import current_app
session_manager = current_app.config['SESSION_MANAGER'] session_manager = current_app.config['SESSION_MANAGER']
return jsonify(session_manager.get_stats()) return success_response(data=session_manager.get_stats())
@auth_bp.route('/health', methods=['GET']) @auth_bp.route('/health', methods=['GET'])
@@ -103,7 +105,7 @@ def get_current_user():
开发模式下支持模拟用户,生产模式下用户信息由后端控制。 开发模式下支持模拟用户,生产模式下用户信息由后端控制。
""" """
user = request.current_user user = request.current_user
return jsonify({ return success_response(data={
"user_id": user["user_id"], "user_id": user["user_id"],
"username": user["username"], "username": user["username"],
"role": user["role"], "role": user["role"],
@@ -133,7 +135,7 @@ def get_users():
""" """
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false' dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
if not dev_mode: if not dev_mode:
return jsonify({"error": "仅开发环境可用"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
users = [] users = []
for username, info in MOCK_USERS.items(): for username, info in MOCK_USERS.items():
@@ -145,7 +147,7 @@ def get_users():
"is_active": True # 模拟用户默认都是活跃状态 "is_active": True # 模拟用户默认都是活跃状态
}) })
return jsonify({"users": users}) return success_response(data={"users": users})
@auth_bp.route('/auth/users/<user_id>', methods=['PUT']) @auth_bp.route('/auth/users/<user_id>', methods=['PUT'])
@@ -161,10 +163,10 @@ def update_user(user_id):
""" """
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false' dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
if not dev_mode: if not dev_mode:
return jsonify({"error": "仅开发环境可用"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
# 模拟用户不支持真正的状态切换,直接返回成功 # 模拟用户不支持真正的状态切换,直接返回成功
return jsonify({"message": "操作成功(模拟)", "user_id": user_id}) return success_response(data={"message": "操作成功(模拟)", "user_id": user_id})
@auth_bp.route('/auth/change-password', methods=['POST']) @auth_bp.route('/auth/change-password', methods=['POST'])
@@ -181,17 +183,17 @@ def change_password():
""" """
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false' dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
if not dev_mode: if not dev_mode:
return jsonify({"error": "仅开发环境可用"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
data = request.json or {} data = request.json or {}
old_password = data.get('old_password') old_password = data.get('old_password')
new_password = data.get('new_password') new_password = data.get('new_password')
if not old_password or not new_password: if not old_password or not new_password:
return jsonify({"error": "请提供旧密码和新密码"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "请提供旧密码和新密码", http_status=400)
if len(new_password) < 6: if len(new_password) < 6:
return jsonify({"error": "新密码至少6位"}), 400 return error_response("INVALID_PARAMS", BAD_REQUEST, "新密码至少6位", http_status=400)
# 模拟环境直接返回成功 # 模拟环境直接返回成功
return jsonify({"message": "密码修改成功(模拟)"}) return success_response(message="密码修改成功(模拟)")

View File

@@ -37,6 +37,8 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
from core.status_codes import SUCCESS, BAD_REQUEST, NOT_FOUND, INTERNAL_ERROR
from api.response_utils import success_response, error_response
from auth.security import validate_query, filter_response from auth.security import validate_query, filter_response
from config import RAG_CHAT_MODEL from config import RAG_CHAT_MODEL
@@ -1238,12 +1240,12 @@ def chat():
history = data.get('history', []) history = data.get('history', [])
if not message: if not message:
return jsonify({"error": "缺少 message"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "缺少 message", http_status=400)
# 输入安全验证 # 输入安全验证
is_valid, reason = validate_query(message) is_valid, reason = validate_query(message)
if not is_valid: if not is_valid:
return jsonify({"error": reason}), 400 return error_response("INVALID_QUERY", BAD_REQUEST, reason, http_status=400)
# 智能聊天 # 智能聊天
result = chat_with_llm(message, history) result = chat_with_llm(message, history)
@@ -1251,7 +1253,7 @@ def chat():
# 过滤敏感信息 # 过滤敏感信息
answer = filter_response(result["answer"]) answer = filter_response(result["answer"])
return jsonify({ return success_response(data={
"answer": answer, "answer": answer,
"mode": "chat", "mode": "chat",
"sources": result.get("sources", []), "sources": result.get("sources", []),
@@ -1309,19 +1311,16 @@ def rag():
session_id = data.get('session_id') session_id = data.get('session_id')
if not message: if not message:
return jsonify({"error": "缺少 message"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "缺少 message", http_status=400)
# 生产环境强制校验 chat_history # 生产环境强制校验 chat_history
if IS_PROD and history is None: if IS_PROD and history is None:
return jsonify({ return error_response("MISSING_HISTORY", BAD_REQUEST, "chat_history is required in production", http_status=400)
"error": "chat_history is required in production",
"code": "MISSING_HISTORY"
}), 400
# 输入安全验证 # 输入安全验证
is_valid, reason = validate_query(message) is_valid, reason = validate_query(message)
if not is_valid: if not is_valid:
return jsonify({"error": reason}), 400 return error_response("INVALID_QUERY", BAD_REQUEST, reason, http_status=400)
# 如果没有指定 collections使用默认的公开库 # 如果没有指定 collections使用默认的公开库
if not collections: if not collections:
@@ -2048,12 +2047,12 @@ def search():
collections = data.get('collections') # 后端传入的知识库列表 collections = data.get('collections') # 后端传入的知识库列表
if not query: if not query:
return jsonify({'error': 'query is required'}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "query is required", http_status=400)
# 输入安全校验(注入检测、违禁词、长度限制) # 输入安全校验(注入检测、违禁词、长度限制)
is_valid, reason = validate_query(query) is_valid, reason = validate_query(query)
if not is_valid: if not is_valid:
return jsonify({'error': reason}), 400 return error_response("INVALID_QUERY", BAD_REQUEST, reason, http_status=400)
# top_k 范围校验 # top_k 范围校验
try: try:
@@ -2067,7 +2066,7 @@ def search():
results = search_hybrid(query, top_k=top_k, allowed_collections=collections) results = search_hybrid(query, top_k=top_k, allowed_collections=collections)
return jsonify({ return success_response(data={
'contexts': results['documents'][0], 'contexts': results['documents'][0],
'metadatas': results['metadatas'][0], 'metadatas': results['metadatas'][0],
'scores': results['scores'][0] 'scores': results['scores'][0]

View File

@@ -39,16 +39,18 @@ import re
import uuid import uuid
from datetime import datetime from datetime import datetime
from typing import Optional, Tuple, Any, List, Dict from typing import Optional, Tuple, Any, List, Dict
from flask import Blueprint, request, jsonify from flask import Blueprint, request
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
from core.status_codes import ( from core.status_codes import (
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST, UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST, SUCCESS,
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION, NO_FILE, NO_FILE_SELECTED, NO_COLLECTION, NOT_FOUND,
FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR,
SERVICE_UNAVAILABLE, DELETE_SUCCESS, UPDATE_SUCCESS, NO_CONTENT,
FORBIDDEN
) )
from api.response_utils import success_response, error_response from api.response_utils import success_response, error_response
@@ -172,7 +174,7 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
仅在 DEV_MODE=true 时可用 仅在 DEV_MODE=true 时可用
""" """
if os.environ.get('DEV_MODE', 'true').lower() == 'false': if os.environ.get('DEV_MODE', 'true').lower() == 'false':
return jsonify({"error": "仅开发环境可用"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
from config import DOCUMENTS_PATH from config import DOCUMENTS_PATH
from flask import send_from_directory from flask import send_from_directory
@@ -181,9 +183,9 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
try: try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH) filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError: except ValueError:
return jsonify({"error": "非法路径"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath): if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
directory = os.path.dirname(filepath) directory = os.path.dirname(filepath)
filename = os.path.basename(filepath) filename = os.path.basename(filepath)
@@ -334,10 +336,24 @@ def upload_document() -> Tuple[Any, int]:
except Exception as e: except Exception as e:
logger.warning(f"标记旧版本失败: {e}") logger.warning(f"标记旧版本失败: {e}")
# 6. 触发向量化 # 6. 触发向量化(异步任务)
sync_status = "已保存,等待手动同步" sync_status = "已保存,等待手动同步"
sync_service = _get_sync_service() sync_service = _get_sync_service()
task_id = None
if sync_service: if sync_service:
from core.task_registry import get_registry
registry = get_registry()
task = registry.create_task('upload', f"向量化: {filename}")
task_id = task.id
def _do_vectorize(task, svc, change_obj, fname):
"""后台执行向量化"""
registry.update_progress(task.id, stage='向量化', message=f"正在处理: {fname}")
svc.process_change(change_obj)
return {'filename': fname, 'sync_status': '已添加到向量库'}
try: try:
from knowledge.sync import DocumentChange, ChangeType from knowledge.sync import DocumentChange, ChangeType
change = DocumentChange( change = DocumentChange(
@@ -348,11 +364,12 @@ def upload_document() -> Tuple[Any, int]:
new_hash=sync_service.calculate_file_hash(filepath), new_hash=sync_service.calculate_file_hash(filepath),
change_time=datetime.now() change_time=datetime.now()
) )
sync_service.process_change(change) registry.start_task(task.id, _do_vectorize, sync_service, change, filename)
sync_status = "已保存并添加到向量库" sync_status = "已保存,向量化任务已启动"
except Exception as e: except Exception as e:
logger.warning(f"向量化失败: {e}") logger.warning(f"创建向量化任务失败: {e}")
sync_status = "已保存,向量化失败" registry.fail_task(task.id, str(e))
sync_status = "已保存,向量化任务创建失败"
return success_response( return success_response(
data={ data={
@@ -363,7 +380,8 @@ def upload_document() -> Tuple[Any, int]:
"size": file_size, "size": file_size,
"replaced": replaced "replaced": replaced
}, },
"sync_status": sync_status "sync_status": sync_status,
"task_id": task_id
}, },
status_code=UPLOAD_SUCCESS, status_code=UPLOAD_SUCCESS,
message=f"文件上传成功,{sync_status}" message=f"文件上传成功,{sync_status}"
@@ -506,14 +524,55 @@ def batch_upload_documents() -> Tuple[Any, int]:
"message": "上传处理失败" "message": "上传处理失败"
}) })
success_count = len([r for r in results if r["status"] == "success"])
task_id = None
# 批量上传完成后自动触发向量化(异步任务)
sync_service = _get_sync_service()
if sync_service and success_count > 0:
from core.task_registry import get_registry
registry = get_registry()
running = registry.list_tasks(status='running', task_type='batch_upload', limit=1)
if not running:
task = registry.create_task('batch_upload', f"批量向量化: {success_count} 个文件", total=success_count)
task_id = task.id
def _do_batch_vectorize(task, svc, count):
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id, current=processed[0], total=count,
stage='批量向量化',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_cb = svc.on_change_callback
svc.on_change_callback = on_change
try:
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
result = svc.sync_now()
return {
'synced': result.documents_processed,
'added': result.documents_added,
'errors': result.errors,
}
finally:
svc.on_change_callback = old_cb
registry.start_task(task.id, _do_batch_vectorize, sync_service, success_count)
return success_response( return success_response(
data={ data={
"total": len(results), "total": len(results),
"success_count": len([r for r in results if r["status"] == "success"]), "success_count": success_count,
"results": results "results": results,
"task_id": task_id
}, },
status_code=BATCH_UPLOAD_SUCCESS, status_code=BATCH_UPLOAD_SUCCESS,
message=f"批量上传完成,成功 {len([r for r in results if r['status'] == 'success'])}/{len(results)} 个文件" message=f"批量上传完成,成功 {success_count}/{len(results)} 个文件"
) )
@@ -590,10 +649,7 @@ def list_documents() -> Tuple[Any, int]:
# 按修改时间倒序 # 按修改时间倒序
documents.sort(key=lambda x: x['last_modified'], reverse=True) documents.sort(key=lambda x: x['last_modified'], reverse=True)
return jsonify({ return success_response(data={"documents": documents, "total": len(documents)})
"documents": documents,
"total": len(documents)
})
@document_bp.route('/documents/<path:doc_path>/status', methods=['GET']) @document_bp.route('/documents/<path:doc_path>/status', methods=['GET'])
@@ -617,12 +673,12 @@ def get_document_status(doc_path: str) -> Tuple[Any, int]:
""" """
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径 # 解析路径
parts = doc_path.split('/') parts = doc_path.split('/')
if len(parts) < 2: if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0] subdir = parts[0]
filename = '/'.join(parts[1:]) filename = '/'.join(parts[1:])
@@ -638,16 +694,14 @@ def get_document_status(doc_path: str) -> Tuple[Any, int]:
if not doc_info: if not doc_info:
if file_on_disk: if file_on_disk:
return jsonify({ return success_response(data={
"success": True,
"status": "unprocessed", "status": "unprocessed",
"chunk_count": 0, "chunk_count": 0,
"last_processed": None "last_processed": None
}) })
return jsonify({"error": "文档不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "文档不存在", http_status=404)
return jsonify({ return success_response(data={
"success": True,
"status": doc_info.get("status", "unknown"), "status": doc_info.get("status", "unknown"),
"chunk_count": doc_info.get("total_chunks", 0), "chunk_count": doc_info.get("total_chunks", 0),
"last_processed": doc_info.get("effective_date") or doc_info.get("version") "last_processed": doc_info.get("effective_date") or doc_info.get("version")
@@ -674,35 +728,35 @@ def update_document(doc_path: str) -> Tuple[Any, int]:
from config import DOCUMENTS_PATH from config import DOCUMENTS_PATH
if 'file' not in request.files: if 'file' not in request.files:
return jsonify({"error": "没有上传文件"}), 400 return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
file = request.files['file'] file = request.files['file']
if file.filename == '': if file.filename == '':
return jsonify({"error": "没有选择文件"}), 400 return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
# 文件类型校验 # 文件类型校验
ext = os.path.splitext(file.filename)[1].lower() ext = os.path.splitext(file.filename)[1].lower()
if ext not in ALLOWED_EXTENSIONS: if ext not in ALLOWED_EXTENSIONS:
return jsonify({"error": f"不支持的文件类型: {ext}"}), 400 return error_response("UNSUPPORTED_FORMAT", UNSUPPORTED_FORMAT, f"不支持的文件类型: {ext}", http_status=400)
# 文件大小校验 # 文件大小校验
file.seek(0, 2) # 跳到文件末尾获取大小 file.seek(0, 2) # 跳到文件末尾获取大小
file_size = file.tell() file_size = file.tell()
file.seek(0) # 回到文件开头 file.seek(0) # 回到文件开头
if file_size > MAX_FILE_SIZE: if file_size > MAX_FILE_SIZE:
return jsonify({"error": f"文件过大(最大 {MAX_FILE_SIZE // 1024 // 1024}MB"}), 400 return error_response("FILE_TOO_LARGE", FILE_TOO_LARGE, f"文件过大(最大 {MAX_FILE_SIZE // 1024 // 1024}MB", http_status=400)
# 解析路径 # 解析路径
parts = doc_path.split('/') parts = doc_path.split('/')
if len(parts) < 2: if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
try: try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH) filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError: except ValueError:
return jsonify({"error": "非法路径"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath): if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
# 覆盖文件 # 覆盖文件
file.save(filepath) file.save(filepath)
@@ -726,10 +780,7 @@ def update_document(doc_path: str) -> Tuple[Any, int]:
except Exception as e: except Exception as e:
logger.warning(f"重新向量化失败: {e}") logger.warning(f"重新向量化失败: {e}")
return jsonify({ return success_response(message="文件已更新")
"success": True,
"message": "文件已更新"
})
@document_bp.route('/documents/<path:doc_path>', methods=['DELETE']) @document_bp.route('/documents/<path:doc_path>', methods=['DELETE'])
@@ -754,7 +805,7 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
# 解析路径 # 解析路径
parts = doc_path.split('/') parts = doc_path.split('/')
if len(parts) < 2: if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0] subdir = parts[0]
filename = '/'.join(parts[1:]) filename = '/'.join(parts[1:])
@@ -765,9 +816,9 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
try: try:
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH) filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
except ValueError: except ValueError:
return jsonify({"error": "非法路径"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
if not os.path.exists(filepath): if not os.path.exists(filepath):
return jsonify({"error": "文件不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
try: try:
# 1. 从向量库删除source 存的是文件名,不是完整路径) # 1. 从向量库删除source 存的是文件名,不是完整路径)
@@ -778,14 +829,11 @@ def delete_document(doc_path: str) -> Tuple[Any, int]:
# 2. 删除文件 # 2. 删除文件
os.remove(filepath) os.remove(filepath)
return jsonify({ return success_response(status_code=DELETE_SUCCESS, message="文档已删除")
"success": True,
"message": "文档已删除"
})
except Exception as e: except Exception as e:
logger.error(f"删除文档异常: {e}") logger.error(f"删除文档异常: {e}")
return jsonify({"error": "删除失败,请稍后重试"}), 500 return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败,请稍后重试", http_status=500)
@document_bp.route('/documents/<path:doc_path>/chunks', methods=['GET']) @document_bp.route('/documents/<path:doc_path>/chunks', methods=['GET'])
@@ -810,12 +858,12 @@ def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
""" """
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径 # 解析路径
parts = doc_path.split('/') parts = doc_path.split('/')
if len(parts) < 2: if len(parts) < 2:
return jsonify({"error": "无效的文档路径"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
subdir = parts[0] subdir = parts[0]
# 目录名即向量库名 # 目录名即向量库名
@@ -823,8 +871,7 @@ def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
chunks = kb_manager.get_document_chunks(collection, os.path.basename(doc_path)) chunks = kb_manager.get_document_chunks(collection, os.path.basename(doc_path))
return jsonify({ return success_response(data={
"success": True,
"document_id": doc_path, "document_id": doc_path,
"collection": collection, "collection": collection,
"chunks": chunks, "chunks": chunks,
@@ -860,12 +907,12 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
""" """
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 解析路径 # 解析路径
parts = doc_path.split('/') parts = doc_path.split('/')
if len(parts) < 2: if len(parts) < 2:
return jsonify({"error": "无效的文档路径,格式: collection/filename"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径,格式: collection/filename", http_status=400)
collection = parts[0] collection = parts[0]
filename = os.path.basename(doc_path) filename = os.path.basename(doc_path)
@@ -880,7 +927,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
# 获取所有切片 # 获取所有切片
all_chunks = kb_manager.get_document_chunks(collection, filename) all_chunks = kb_manager.get_document_chunks(collection, filename)
if not all_chunks: if not all_chunks:
return jsonify({"error": f"文档 '{filename}' 不存在或无切片"}), 404 return error_response("NOT_FOUND", NOT_FOUND, f"文档 '{filename}' 不存在或无切片", http_status=404)
total = len(all_chunks) total = len(all_chunks)
@@ -889,8 +936,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
preview_chunks = all_chunks[:5] preview_chunks = all_chunks[:5]
for c in preview_chunks: for c in preview_chunks:
c['is_target'] = False c['is_target'] = False
return jsonify({ return success_response(data={
"success": True,
"collection": collection, "collection": collection,
"source": filename, "source": filename,
"total_chunks": total, "total_chunks": total,
@@ -902,7 +948,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
try: try:
target_chunk_index = int(chunk_index_str) target_chunk_index = int(chunk_index_str)
except ValueError: except ValueError:
return jsonify({"error": "chunk_index 必须为整数"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "chunk_index 必须为整数", http_status=400)
# 按 chunk_index 排序Chroma 返回顺序不保证有序) # 按 chunk_index 排序Chroma 返回顺序不保证有序)
all_chunks.sort(key=lambda c: c.get('metadata', {}).get('chunk_index', 0)) all_chunks.sort(key=lambda c: c.get('metadata', {}).get('chunk_index', 0))
@@ -923,9 +969,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
(c.get('metadata', {}).get('chunk_index', 0) for c in all_chunks), (c.get('metadata', {}).get('chunk_index', 0) for c in all_chunks),
default=total - 1 default=total - 1
) )
return jsonify({ return error_response("BAD_REQUEST", BAD_REQUEST, f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})", http_status=400)
"error": f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})"
}), 400
# 截取上下文窗口 # 截取上下文窗口
start = max(0, target_pos - context_count) start = max(0, target_pos - context_count)
@@ -936,8 +980,7 @@ def preview_document(doc_path: str) -> Tuple[Any, int]:
for i, c in enumerate(window): for i, c in enumerate(window):
c['is_target'] = (start + i == target_pos) c['is_target'] = (start + i == target_pos)
return jsonify({ return success_response(data={
"success": True,
"collection": collection, "collection": collection,
"source": filename, "source": filename,
"total_chunks": total, "total_chunks": total,
@@ -968,7 +1011,7 @@ def create_chunk() -> Tuple[Any, int]:
""" """
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
data = request.json or {} data = request.json or {}
collection = data.get('collection') collection = data.get('collection')
@@ -976,17 +1019,13 @@ def create_chunk() -> Tuple[Any, int]:
metadata = data.get('metadata', {}) metadata = data.get('metadata', {})
if not collection: if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400 return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
if not content: if not content:
return jsonify({"error": "切片内容不能为空"}), 400 return error_response("NO_CONTENT", NO_CONTENT, "切片内容不能为空", http_status=400)
chunk_id = kb_manager.add_chunk(collection, content, metadata) chunk_id = kb_manager.add_chunk(collection, content, metadata)
return jsonify({ return success_response(data={"chunk_id": chunk_id}, message="切片已添加")
"success": True,
"chunk_id": chunk_id,
"message": "切片已添加"
})
@document_bp.route('/chunks/<chunk_id>', methods=['PUT']) @document_bp.route('/chunks/<chunk_id>', methods=['PUT'])
@@ -1012,7 +1051,7 @@ def update_chunk(chunk_id: str) -> Tuple[Any, int]:
""" """
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
data = request.json or {} data = request.json or {}
collection = data.get('collection') collection = data.get('collection')
@@ -1020,13 +1059,13 @@ def update_chunk(chunk_id: str) -> Tuple[Any, int]:
metadata = data.get('metadata') metadata = data.get('metadata')
if not collection: if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400 return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
success = kb_manager.update_chunk(collection, chunk_id, content=content, metadata=metadata) success = kb_manager.update_chunk(collection, chunk_id, content=content, metadata=metadata)
if success: if success:
return jsonify({"success": True, "message": "切片已更新"}) return success_response(message="切片已更新")
return jsonify({"error": "更新失败"}), 500 return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "更新失败", http_status=500)
@document_bp.route('/chunks/<chunk_id>', methods=['DELETE']) @document_bp.route('/chunks/<chunk_id>', methods=['DELETE'])
@@ -1054,11 +1093,11 @@ def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
collection = request.args.get('collection') collection = request.args.get('collection')
if not collection: if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400 return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 删除切片,返回 (success, source_file) # 删除切片,返回 (success, source_file)
success, source_file = kb_manager.delete_chunk(collection, chunk_id) success, source_file = kb_manager.delete_chunk(collection, chunk_id)
@@ -1083,8 +1122,8 @@ def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
logger.warning(f"清理哈希记录失败: {e}") logger.warning(f"清理哈希记录失败: {e}")
if success: if success:
return jsonify({"success": True, "message": "切片已删除"}) return success_response(status_code=DELETE_SUCCESS, message="切片已删除")
return jsonify({"error": "删除失败"}), 500 return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败", http_status=500)
@document_bp.route('/chunks/batch', methods=['DELETE']) @document_bp.route('/chunks/batch', methods=['DELETE'])
@@ -1110,13 +1149,13 @@ def delete_chunks_by_source() -> Tuple[Any, int]:
source = data.get('source') source = data.get('source')
if not collection: if not collection:
return jsonify({"error": "请指定向量库 (collection)"}), 400 return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
if not source: if not source:
return jsonify({"error": "请指定文件名 (source)"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "请指定文件名 (source)", http_status=400)
kb_manager = _get_kb_manager() kb_manager = _get_kb_manager()
if not kb_manager: if not kb_manager:
return jsonify({"error": "知识库管理器未初始化"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
# 批量删除该文件的所有切片 # 批量删除该文件的所有切片
try: try:
@@ -1134,11 +1173,7 @@ def delete_chunks_by_source() -> Tuple[Any, int]:
except Exception as e: except Exception as e:
logger.warning(f"清理哈希记录失败: {e}") logger.warning(f"清理哈希记录失败: {e}")
return jsonify({ return success_response(data={"deleted_count": deleted_count}, message=f"已删除 {deleted_count} 个切片")
"success": True,
"deleted_count": deleted_count,
"message": f"已删除 {deleted_count} 个切片"
})
except Exception as e: except Exception as e:
logger.error(f"操作异常: {e}") logger.error(f"操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)

View File

@@ -33,6 +33,11 @@
from flask import Blueprint, request, jsonify from flask import Blueprint, request, jsonify
from auth.gateway import require_gateway_auth, require_role from auth.gateway import require_gateway_auth, require_role
from core.status_codes import (
SUCCESS, CREATED, DELETE_SUCCESS, UPDATE_SUCCESS,
BAD_REQUEST, NOT_FOUND, INTERNAL_ERROR
)
from api.response_utils import success_response, error_response
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -75,10 +80,10 @@ def submit_feedback():
user_id = data.get('user_id', '') user_id = data.get('user_id', '')
if not session_id or not query or rating is None: if not session_id or not query or rating is None:
return jsonify({"error": "缺少必要参数"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "缺少必要参数", http_status=400)
if rating not in [1, -1]: if rating not in [1, -1]:
return jsonify({"error": "rating 必须是 1 或 -1"}), 400 return error_response("INVALID_PARAMS", BAD_REQUEST, "rating 必须是 1 或 -1", http_status=400)
try: try:
feedback_service = _get_feedback_service() feedback_service = _get_feedback_service()
@@ -91,15 +96,14 @@ def submit_feedback():
reason=reason, reason=reason,
user_id=user_id user_id=user_id
) )
return jsonify({ return success_response(data={
"success": True,
"feedback_id": result['feedback_id'], "feedback_id": result['feedback_id'],
"faq_suggested": result.get('faq_suggested', False), "faq_suggested": result.get('faq_suggested', False),
"suggestion_id": result.get('suggestion_id') "suggestion_id": result.get('suggestion_id')
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/feedback/stats', methods=['GET']) @feedback_bp.route('/feedback/stats', methods=['GET'])
@@ -112,13 +116,12 @@ def get_feedback_stats():
try: try:
feedback_db = _get_feedback_db() feedback_db = _get_feedback_db()
stats = feedback_db.get_feedback_stats(start_date, end_date) stats = feedback_db.get_feedback_stats(start_date, end_date)
return jsonify({ return success_response(data={
"success": True,
"stats": stats "stats": stats
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/feedback/list', methods=['GET']) @feedback_bp.route('/feedback/list', methods=['GET'])
@@ -140,14 +143,13 @@ def get_feedback_list():
end_date=end_date, end_date=end_date,
limit=limit limit=limit
) )
return jsonify({ return success_response(data={
"success": True,
"feedbacks": feedbacks, "feedbacks": feedbacks,
"total": len(feedbacks) "total": len(feedbacks)
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/reports/weekly', methods=['GET']) @feedback_bp.route('/reports/weekly', methods=['GET'])
@@ -157,13 +159,12 @@ def get_weekly_report():
try: try:
feedback_service = _get_feedback_service() feedback_service = _get_feedback_service()
report = feedback_service.generate_report("weekly") report = feedback_service.generate_report("weekly")
return jsonify({ return success_response(data={
"success": True,
"report": report.to_dict() "report": report.to_dict()
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/reports/monthly', methods=['GET']) @feedback_bp.route('/reports/monthly', methods=['GET'])
@@ -173,13 +174,12 @@ def get_monthly_report():
try: try:
feedback_service = _get_feedback_service() feedback_service = _get_feedback_service()
report = feedback_service.generate_report("monthly") report = feedback_service.generate_report("monthly")
return jsonify({ return success_response(data={
"success": True,
"report": report.to_dict() "report": report.to_dict()
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq', methods=['GET']) @feedback_bp.route('/faq', methods=['GET'])
@@ -192,14 +192,13 @@ def get_faq_list():
try: try:
feedback_db = _get_feedback_db() feedback_db = _get_feedback_db()
faqs = feedback_db.get_faqs(status=status, limit=limit) faqs = feedback_db.get_faqs(status=status, limit=limit)
return jsonify({ return success_response(data={
"success": True,
"faqs": faqs, "faqs": faqs,
"total": len(faqs) "total": len(faqs)
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq', methods=['POST']) @feedback_bp.route('/faq', methods=['POST'])
@@ -220,7 +219,7 @@ def create_faq():
answer = data.get('answer') answer = data.get('answer')
if not question or not answer: if not question or not answer:
return jsonify({"error": "缺少问题或答案"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "缺少问题或答案", http_status=400)
try: try:
from services.feedback import FAQ from services.feedback import FAQ
@@ -235,15 +234,14 @@ def create_faq():
) )
faq_id = feedback_db.add_faq(faq) faq_id = feedback_db.add_faq(faq)
return jsonify({ return success_response(data={
"success": True,
"faq_id": faq_id, "faq_id": faq_id,
"status": "draft", "status": "draft",
"message": "FAQ已创建请通过 /faq/<id>/approve 接口确认后生效" "message": "FAQ已创建请通过 /faq/<id>/approve 接口确认后生效"
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/<int:faq_id>/approve', methods=['POST']) @feedback_bp.route('/faq/<int:faq_id>/approve', methods=['POST'])
@@ -263,10 +261,10 @@ def approve_faq(faq_id):
# 检查 FAQ 状态 # 检查 FAQ 状态
faq = feedback_db.get_faq(faq_id) faq = feedback_db.get_faq(faq_id)
if not faq: if not faq:
return jsonify({"error": "FAQ不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "FAQ不存在", http_status=404)
if faq.get('status') == 'approved': if faq.get('status') == 'approved':
return jsonify({"success": True, "message": "FAQ已经是批准状态"}) return success_response(message="FAQ已经是批准状态")
# 更新状态为 approved # 更新状态为 approved
feedback_db.update_faq(faq_id, {"status": "approved"}) feedback_db.update_faq(faq_id, {"status": "approved"})
@@ -279,15 +277,14 @@ def approve_faq(faq_id):
answer=faq['answer'] answer=faq['answer']
) )
return jsonify({ return success_response(data={
"success": True,
"faq_id": faq_id, "faq_id": faq_id,
"sync_status": "synced" if sync_success else "sync_failed", "sync_status": "synced" if sync_success else "sync_failed",
"message": "FAQ已批准并同步到知识库" "message": "FAQ已批准并同步到知识库"
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/<int:faq_id>', methods=['PUT']) @feedback_bp.route('/faq/<int:faq_id>', methods=['PUT'])
@@ -303,11 +300,11 @@ def update_faq(faq_id):
# 获取更新前的 FAQ 信息(用于判断是否需要重新同步) # 获取更新前的 FAQ 信息(用于判断是否需要重新同步)
old_faq = feedback_db.get_faq(faq_id) old_faq = feedback_db.get_faq(faq_id)
if not old_faq: if not old_faq:
return jsonify({"error": "FAQ不存在"}), 404 return error_response("NOT_FOUND", NOT_FOUND, "FAQ不存在", http_status=404)
updated = feedback_db.update_faq(faq_id, data) updated = feedback_db.update_faq(faq_id, data)
if not updated: if not updated:
return jsonify({"error": "FAQ更新失败"}), 500 return error_response("UPDATE_FAILED", INTERNAL_ERROR, "FAQ更新失败", http_status=500)
# 检查是否需要重新同步向量库question 或 answer 变更时) # 检查是否需要重新同步向量库question 或 answer 变更时)
need_sync = False need_sync = False
@@ -331,14 +328,13 @@ def update_faq(faq_id):
) )
sync_status = "synced" if sync_success else "sync_failed" sync_status = "synced" if sync_success else "sync_failed"
return jsonify({ return success_response(data={
"success": True,
"message": "FAQ更新成功", "message": "FAQ更新成功",
"sync_status": sync_status "sync_status": sync_status
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/<int:faq_id>', methods=['DELETE']) @feedback_bp.route('/faq/<int:faq_id>', methods=['DELETE'])
@@ -365,13 +361,10 @@ def delete_faq(faq_id):
feedback_service = _get_feedback_service() feedback_service = _get_feedback_service()
feedback_service._delete_faq_vectors(faq_id) feedback_service._delete_faq_vectors(faq_id)
return jsonify({ return success_response(data={"deleted": deleted}, message="FAQ删除成功" if deleted else "FAQ不存在")
"success": deleted,
"message": "FAQ删除成功" if deleted else "FAQ不存在"
})
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/suggestions', methods=['GET']) @feedback_bp.route('/faq/suggestions', methods=['GET'])
@@ -385,14 +378,13 @@ def get_faq_suggestions():
try: try:
feedback_db = _get_feedback_db() feedback_db = _get_feedback_db()
suggestions = feedback_db.get_faq_suggestions(status=status, limit=limit) suggestions = feedback_db.get_faq_suggestions(status=status, limit=limit)
return jsonify({ return success_response(data={
"success": True,
"suggestions": suggestions, "suggestions": suggestions,
"total": len(suggestions) "total": len(suggestions)
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/suggestions/<int:suggestion_id>/approve', methods=['POST']) @feedback_bp.route('/faq/suggestions/<int:suggestion_id>/approve', methods=['POST'])
@@ -418,17 +410,16 @@ def approve_faq_suggestion(suggestion_id):
) )
if result.get('success'): if result.get('success'):
return jsonify({ return success_response(data={
"success": True,
"faq_id": result['faq_id'], "faq_id": result['faq_id'],
"sync_status": result.get('sync_status'), "sync_status": result.get('sync_status'),
"message": "FAQ建议已批准并同步到知识库" "message": "FAQ建议已批准并同步到知识库"
}) })
else: else:
return jsonify({"error": result.get('error', '批准失败')}), 400 return error_response("APPROVE_FAILED", BAD_REQUEST, result.get('error', '批准失败'), http_status=400)
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/faq/suggestions/<int:suggestion_id>/reject', methods=['POST']) @feedback_bp.route('/faq/suggestions/<int:suggestion_id>/reject', methods=['POST'])
@@ -439,13 +430,13 @@ def reject_faq_suggestion(suggestion_id):
try: try:
feedback_db = _get_feedback_db() feedback_db = _get_feedback_db()
rejected = feedback_db.reject_faq_suggestion(suggestion_id) rejected = feedback_db.reject_faq_suggestion(suggestion_id)
return jsonify({ return success_response(data={
"success": rejected, "rejected": rejected,
"message": "FAQ建议已拒绝" if rejected else "建议不存在" "message": "FAQ建议已拒绝" if rejected else "建议不存在"
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
# ==================== Bad Case 分析接口 ==================== # ==================== Bad Case 分析接口 ====================
@@ -477,8 +468,7 @@ def get_bad_cases():
for case in bad_cases: for case in bad_cases:
case['status'] = 'pending' # pending/resolved/ignored case['status'] = 'pending' # pending/resolved/ignored
return jsonify({ return success_response(data={
"success": True,
"bad_cases": bad_cases, "bad_cases": bad_cases,
"blacklisted_sources": blacklisted_sources, "blacklisted_sources": blacklisted_sources,
"suggestions": [ "suggestions": [
@@ -489,7 +479,7 @@ def get_bad_cases():
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@feedback_bp.route('/feedback/blacklist', methods=['GET']) @feedback_bp.route('/feedback/blacklist', methods=['GET'])
@@ -507,12 +497,11 @@ def get_chunk_blacklist():
feedback_service = _get_feedback_service() feedback_service = _get_feedback_service()
blacklist = feedback_service.get_chunk_blacklist(min_dislikes=min_dislikes) blacklist = feedback_service.get_chunk_blacklist(min_dislikes=min_dislikes)
return jsonify({ return success_response(data={
"success": True,
"blacklist": list(blacklist), "blacklist": list(blacklist),
"count": len(blacklist), "count": len(blacklist),
"usage": "在检索时过滤这些来源以提升回答质量" "usage": "在检索时过滤这些来源以提升回答质量"
}) })
except Exception as e: except Exception as e:
logger.error(f"反馈操作异常: {e}") logger.error(f"反馈操作异常: {e}")
return jsonify({"error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)

View File

@@ -11,6 +11,8 @@
import os import os
import logging import logging
from flask import Blueprint, send_file, jsonify, current_app from flask import Blueprint, send_file, jsonify, current_app
from core.status_codes import SUCCESS, BAD_REQUEST, NOT_FOUND, INTERNAL_ERROR
from api.response_utils import success_response, error_response
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -51,7 +53,7 @@ def get_image(image_id: str):
# 安全检查:防止路径遍历攻击 # 安全检查:防止路径遍历攻击
if '..' in image_id or '/' in image_id or '\\' in image_id: if '..' in image_id or '/' in image_id or '\\' in image_id:
return jsonify({"error": "无效的图片 ID"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的图片 ID", http_status=400)
images_path = get_images_base_path() images_path = get_images_base_path()
@@ -74,9 +76,9 @@ def get_image(image_id: str):
return send_file(image_path, mimetype=mimetype) return send_file(image_path, mimetype=mimetype)
except Exception as e: except Exception as e:
logger.error(f"读取图片异常: {e}") logger.error(f"读取图片异常: {e}")
return jsonify({"error": "读取图片失败"}), 500 return error_response("READ_ERROR", INTERNAL_ERROR, "读取图片失败", http_status=500)
return jsonify({"error": "图片不存在", "image_id": image_id}), 404 return error_response("NOT_FOUND", NOT_FOUND, "图片不存在", http_status=404, image_id=image_id)
@image_bp.route('/images/<image_id>/info', methods=['GET']) @image_bp.route('/images/<image_id>/info', methods=['GET'])
@@ -96,7 +98,7 @@ def get_image_info(image_id: str):
# 安全检查 # 安全检查
if '..' in image_id or '/' in image_id or '\\' in image_id: if '..' in image_id or '/' in image_id or '\\' in image_id:
return jsonify({"error": "无效的图片 ID"}), 400 return error_response("BAD_REQUEST", BAD_REQUEST, "无效的图片 ID", http_status=400)
images_path = get_images_base_path() images_path = get_images_base_path()
supported_formats = ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'] supported_formats = ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp']
@@ -109,7 +111,7 @@ def get_image_info(image_id: str):
from PIL import Image from PIL import Image
with Image.open(image_path) as img: with Image.open(image_path) as img:
return jsonify({ return success_response(data={
"image_id": image_id, "image_id": image_id,
"width": img.width, "width": img.width,
"height": img.height, "height": img.height,
@@ -120,16 +122,16 @@ def get_image_info(image_id: str):
}) })
except ImportError: except ImportError:
# PIL 未安装,返回基本信息 # PIL 未安装,返回基本信息
return jsonify({ return success_response(data={
"image_id": image_id, "image_id": image_id,
"size_bytes": os.path.getsize(image_path), "size_bytes": os.path.getsize(image_path),
"url": f"/images/{image_id}" "url": f"/images/{image_id}"
}) })
except Exception as e: except Exception as e:
logger.error(f"读取图片信息异常: {e}") logger.error(f"读取图片信息异常: {e}")
return jsonify({"error": "读取图片信息失败"}), 500 return error_response("READ_ERROR", INTERNAL_ERROR, "读取图片信息失败", http_status=500)
return jsonify({"error": "图片不存在", "image_id": image_id}), 404 return error_response("NOT_FOUND", NOT_FOUND, "图片不存在", http_status=404, image_id=image_id)
@image_bp.route('/images/list', methods=['GET']) @image_bp.route('/images/list', methods=['GET'])
@@ -152,7 +154,7 @@ def list_images():
images_path = get_images_base_path() images_path = get_images_base_path()
if not os.path.exists(images_path): if not os.path.exists(images_path):
return jsonify({"images": [], "total": 0}) return success_response(data={"images": [], "total": 0})
supported_extensions = {'.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'} supported_extensions = {'.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'}
images = [] images = []
@@ -176,7 +178,7 @@ def list_images():
total = len(images) total = len(images)
images = images[offset:offset + limit] images = images[offset:offset + limit]
return jsonify({ return success_response(data={
"images": images, "images": images,
"total": total, "total": total,
"limit": limit, "limit": limit,
@@ -185,7 +187,7 @@ def list_images():
except Exception as e: except Exception as e:
logger.error(f"列出图片异常: {e}") logger.error(f"列出图片异常: {e}")
return jsonify({"error": "列出图片失败"}), 500 return error_response("LIST_ERROR", INTERNAL_ERROR, "列出图片失败", http_status=500)
@image_bp.route('/images/stats', methods=['GET']) @image_bp.route('/images/stats', methods=['GET'])
@@ -199,7 +201,7 @@ def image_stats():
images_path = get_images_base_path() images_path = get_images_base_path()
if not os.path.exists(images_path): if not os.path.exists(images_path):
return jsonify({ return success_response(data={
"total_images": 0, "total_images": 0,
"total_size_bytes": 0, "total_size_bytes": 0,
"supported_formats": ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'] "supported_formats": ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp']
@@ -219,7 +221,7 @@ def image_stats():
total_size += os.path.getsize(filepath) total_size += os.path.getsize(filepath)
format_counts[ext] = format_counts.get(ext, 0) + 1 format_counts[ext] = format_counts.get(ext, 0) + 1
return jsonify({ return success_response(data={
"total_images": total_count, "total_images": total_count,
"total_size_bytes": total_size, "total_size_bytes": total_size,
"total_size_mb": round(total_size / (1024 * 1024), 2), "total_size_mb": round(total_size / (1024 * 1024), 2),
@@ -229,4 +231,4 @@ def image_stats():
except Exception as e: except Exception as e:
logger.error(f"获取统计信息异常: {e}") logger.error(f"获取统计信息异常: {e}")
return jsonify({"error": "获取统计信息失败"}), 500 return error_response("STATS_ERROR", INTERNAL_ERROR, "获取统计信息失败", http_status=500)

View File

@@ -37,6 +37,13 @@ from typing import Tuple, Optional, Any
from flask import Blueprint, request, jsonify, current_app from flask import Blueprint, request, jsonify, current_app
import logging import logging
from core.status_codes import (
SUCCESS, CREATED, DELETE_SUCCESS, UPDATE_SUCCESS, SYNC_SUCCESS,
BAD_REQUEST, NOT_FOUND, COLLECTION_NOT_FOUND, NO_COLLECTION, TASK_CONFLICT,
INTERNAL_ERROR, SERVICE_UNAVAILABLE, SYNC_ERROR, REINDEX_ERROR
)
from api.response_utils import success_response, error_response
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
@@ -133,10 +140,7 @@ def list_collections() -> Tuple[Any, int]:
"description": coll.description "description": coll.description
}) })
return jsonify({ return success_response(data={"collections": result, "total": len(result)})
"collections": result,
"total": len(result)
})
@kb_bp.route('/collections', methods=['POST']) @kb_bp.route('/collections', methods=['POST'])
@@ -172,22 +176,19 @@ def create_collection() -> Tuple[Any, int]:
description = data.get('description', '') description = data.get('description', '')
if not name: if not name:
return jsonify({"error": "向量库名称不能为空"}), 400 return error_response("INVALID_NAME", BAD_REQUEST, "向量库名称不能为空", http_status=400)
# 验证名称格式ChromaDB 限制) # 验证名称格式ChromaDB 限制)
if not name.replace('_', '').replace('-', '').isalnum(): if not name.replace('_', '').replace('-', '').isalnum():
return jsonify({ return error_response("INVALID_NAME_FORMAT", BAD_REQUEST, "向量库名称只能包含字母、数字、下划线和连字符", http_status=400)
"error": "名称格式错误",
"message": "向量库名称只能包含字母、数字、下划线和连字符"
}), 400
success, message = kb_manager.create_collection( success, message = kb_manager.create_collection(
name, display_name, department, description name, display_name, department, description
) )
if success: if success:
return jsonify({"success": True, "message": message, "name": name}), 201 return success_response(data={"name": name}, status_code=CREATED, message=message, http_status=201)
return jsonify({"error": message}), 400 return error_response("CREATE_FAILED", BAD_REQUEST, message, http_status=400)
@kb_bp.route('/collections/<kb_name>', methods=['PUT']) @kb_bp.route('/collections/<kb_name>', methods=['PUT'])
@@ -220,7 +221,7 @@ def update_collection(kb_name: str) -> Tuple[Any, int]:
# 检查向量库是否存在 # 检查向量库是否存在
collections = kb_manager.list_collections() collections = kb_manager.list_collections()
if not any(c.name == kb_name for c in collections): if not any(c.name == kb_name for c in collections):
return jsonify({"error": f"向量库 '{kb_name}' 不存在"}), 404 return error_response("COLLECTION_NOT_FOUND", COLLECTION_NOT_FOUND, f"向量库 '{kb_name}' 不存在", http_status=404)
# 更新元数据 # 更新元数据
success = kb_manager.update_collection_metadata( success = kb_manager.update_collection_metadata(
@@ -230,8 +231,8 @@ def update_collection(kb_name: str) -> Tuple[Any, int]:
) )
if success: if success:
return jsonify({"success": True, "message": "向量库信息已更新"}) return success_response(data=None, status_code=UPDATE_SUCCESS, message="向量库信息已更新")
return jsonify({"error": "更新失败"}), 500 return error_response("UPDATE_FAILED", INTERNAL_ERROR, "更新失败", http_status=500)
@kb_bp.route('/collections/<kb_name>', methods=['DELETE']) @kb_bp.route('/collections/<kb_name>', methods=['DELETE'])
@@ -260,12 +261,8 @@ def delete_collection(kb_name: str) -> Tuple[Any, int]:
success, message = kb_manager.delete_collection(kb_name, delete_documents) success, message = kb_manager.delete_collection(kb_name, delete_documents)
if success: if success:
return jsonify({ return success_response(data={"deleted_documents": delete_documents}, status_code=DELETE_SUCCESS, message=message)
"success": True, return error_response("DELETE_FAILED", BAD_REQUEST, message, http_status=400)
"message": message,
"deleted_documents": delete_documents
})
return jsonify({"error": message}), 400
@kb_bp.route('/collections/<kb_name>/documents', methods=['GET']) @kb_bp.route('/collections/<kb_name>/documents', methods=['GET'])
@@ -286,11 +283,7 @@ def list_collection_documents(kb_name: str) -> Tuple[Any, int]:
documents = kb_manager.list_documents(kb_name) documents = kb_manager.list_documents(kb_name)
return jsonify({ return success_response(data={"collection": kb_name, "documents": documents, "total": len(documents)})
"collection": kb_name,
"documents": documents,
"total": len(documents)
})
@kb_bp.route('/collections/<kb_name>/chunks', methods=['GET']) @kb_bp.route('/collections/<kb_name>/chunks', methods=['GET'])
@@ -321,21 +314,16 @@ def list_collection_chunks(kb_name: str) -> Tuple[Any, int]:
chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset) chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset)
return jsonify({ return success_response(data={"collection": kb_name, "chunks": chunks, "total": len(chunks)})
"collection": kb_name,
"chunks": chunks,
"total": len(chunks)
})
@kb_bp.route('/documents/sync', methods=['POST']) @kb_bp.route('/documents/sync', methods=['POST'])
@require_gateway_auth @require_gateway_auth
def sync_documents() -> Tuple[Any, int]: def sync_documents() -> Tuple[Any, int]:
""" """
触发文档向量化同步 触发文档向量化同步(异步任务)
扫描文档目录,检测新增、修改、删除的文件, 立即返回 task_id后台线程执行同步。
自动更新向量库索引。
请求体: 请求体:
{ {
@@ -343,11 +331,7 @@ def sync_documents() -> Tuple[Any, int]:
} }
Returns: Returns:
{ {"success": true, "data": {"task_id": "xxx", "message": "..."}}
"success": true,
"results": [{"collection": "...", "status": "...", ...}],
"synced_count": N
}
""" """
kb_manager, _, err = _require_multi_kb() kb_manager, _, err = _require_multi_kb()
if err: if err:
@@ -355,7 +339,6 @@ def sync_documents() -> Tuple[Any, int]:
from config import DOCUMENTS_PATH from config import DOCUMENTS_PATH
user = request.current_user
data = request.json or {} data = request.json or {}
target_collection = data.get('collection') target_collection = data.get('collection')
@@ -363,54 +346,67 @@ def sync_documents() -> Tuple[Any, int]:
if target_collection: if target_collection:
collections_to_sync = [target_collection] collections_to_sync = [target_collection]
else: else:
# 同步所有向量库
all_collections = kb_manager.list_collections() all_collections = kb_manager.list_collections()
collections_to_sync = [c.name for c in all_collections] collections_to_sync = [c.name for c in all_collections]
if not collections_to_sync: if not collections_to_sync:
return jsonify({"error": "没有可同步的向量库"}), 400 return error_response("NO_COLLECTION", NO_COLLECTION, "没有可同步的向量库", http_status=400)
# 执行同步
results = []
# 使用 sync_service 执行同步
sync_service = current_app.config.get('SYNC_SERVICE') sync_service = current_app.config.get('SYNC_SERVICE')
if not sync_service:
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503,
results=[{"collection": c, "status": "warning", "message": "同步服务不可用"} for c in collections_to_sync],
synced_count=0
)
if sync_service: from core.task_registry import get_registry
registry = get_registry()
# 检查是否有正在运行的同步任务
running = registry.list_tasks(status='running', task_type='sync', limit=1)
if running:
return error_response("TASK_RUNNING", TASK_CONFLICT, f"同步任务正在执行中 (task_id: {running[0].id})", http_status=409)
desc = f"文档同步: {target_collection or '所有向量库'}"
task = registry.create_task('sync', desc)
def _do_sync(task, sync_svc):
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id, current=processed[0],
stage='处理文件',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_callback = sync_svc.on_change_callback
sync_svc.on_change_callback = on_change
try: try:
sync_result = sync_service.sync_now() registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
results.append({ sync_result = sync_svc.sync_now()
"collection": "all", return {
"status": "success", 'collection': target_collection or 'all',
"message": f"同步完成: 处理 {sync_result.documents_processed} 个文档", 'status': 'success',
"details": { 'message': f"同步完成: 处理 {sync_result.documents_processed} 个文档",
"added": sync_result.documents_added, 'details': {
"modified": sync_result.documents_modified, 'added': sync_result.documents_added,
"deleted": sync_result.documents_deleted, 'modified': sync_result.documents_modified,
"errors": sync_result.errors 'deleted': sync_result.documents_deleted,
'errors': sync_result.errors,
} }
}) }
except Exception as e: finally:
logger.error(f"知识库操作异常: {e}") sync_svc.on_change_callback = old_callback
results.append({
"collection": "all",
"status": "error",
"message": "操作失败"
})
else:
# 没有 sync_service返回提示
for coll_name in collections_to_sync:
results.append({
"collection": coll_name,
"status": "warning",
"message": "同步服务不可用,请使用 POST /sync 端点"
})
return jsonify({ registry.start_task(task.id, _do_sync, sync_service)
"success": True,
"results": results, return success_response(
"synced_count": len([r for r in results if r["status"] == "success"]) data={"task_id": task.id, "message": f"同步任务已启动,通过 GET /tasks/{task.id} 查询进度"},
}) status_code=SYNC_SUCCESS,
message="同步任务已启动"
)
@kb_bp.route('/debug/scan', methods=['GET']) @kb_bp.route('/debug/scan', methods=['GET'])
@@ -473,22 +469,16 @@ def debug_scan() -> Tuple[Any, int]:
@require_gateway_auth @require_gateway_auth
def reindex_collection(kb_name: str) -> Tuple[Any, int]: def reindex_collection(kb_name: str) -> Tuple[Any, int]:
""" """
强制重新向量化指定集合的所有文档 强制重新向量化指定集合的所有文档(异步任务)
清除该集合的文档哈希记录,触发完整重新索引。 清除该集合的文档哈希记录,触发完整重新索引。
适用于文档内容更新后需要重建索引的场景 立即返回 task_id后台线程执行重建
Args: Args:
kb_name: 向量库名称 kb_name: 向量库名称
Returns: Returns:
{ {"success": true, "data": {"task_id": "xxx", "message": "..."}}
"success": true,
"message": "...",
"documents_processed": N,
"documents_added": N,
"errors": [...]
}
""" """
kb_manager, _, err = _require_multi_kb() kb_manager, _, err = _require_multi_kb()
if err: if err:
@@ -496,14 +486,12 @@ def reindex_collection(kb_name: str) -> Tuple[Any, int]:
from config import DOCUMENTS_PATH from config import DOCUMENTS_PATH
# 清除该集合的文档哈希记录 # 清除该集合的文档哈希记录(同步完成,很快)
try: try:
from data.db import get_connection from data.db import get_connection
with get_connection("knowledge") as conn: with get_connection("knowledge") as conn:
cursor = conn.cursor() cursor = conn.cursor()
# 转义 LIKE 通配符,防止 kb_name 中的 % 或 _ 导致非预期匹配
escaped_kb = kb_name.replace('%', '\\%').replace('_', '\\_') escaped_kb = kb_name.replace('%', '\\%').replace('_', '\\_')
# 删除以 "{kb_name}/" 或 "{kb_name}\" 开头的文档哈希(兼容 Windows 和 Linux
cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? ESCAPE '\\' OR document_id LIKE ? ESCAPE '\\'", cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? ESCAPE '\\' OR document_id LIKE ? ESCAPE '\\'",
(f"{escaped_kb}/%", f"{escaped_kb}\\%")) (f"{escaped_kb}/%", f"{escaped_kb}\\%"))
deleted = cursor.rowcount deleted = cursor.rowcount
@@ -511,23 +499,54 @@ def reindex_collection(kb_name: str) -> Tuple[Any, int]:
except Exception as e: except Exception as e:
logger.warning(f"清除哈希记录失败: {e}") logger.warning(f"清除哈希记录失败: {e}")
# 触发同步
sync_service = current_app.config.get('SYNC_SERVICE') sync_service = current_app.config.get('SYNC_SERVICE')
if sync_service: if not sync_service:
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503)
from core.task_registry import get_registry
registry = get_registry()
# 检查是否有正在运行的重建任务
running = registry.list_tasks(status='running', task_type='reindex', limit=1)
if running:
return error_response("TASK_RUNNING", TASK_CONFLICT, f"重建任务正在执行中 (task_id: {running[0].id})", http_status=409)
task = registry.create_task('reindex', f'重建索引: {kb_name}')
def _do_reindex(task, sync_svc, kb):
"""后台执行重建索引"""
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id,
current=processed[0],
stage='重新索引',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_callback = sync_svc.on_change_callback
sync_svc.on_change_callback = on_change
try: try:
result = sync_service.sync_now() registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
return jsonify({ result = sync_svc.sync_now()
"success": True, return {
"message": f"重新索引完成: 处理 {result.documents_processed} 个文档", 'message': f"重新索引完成: 处理 {result.documents_processed} 个文档",
"documents_processed": result.documents_processed, 'documents_processed': result.documents_processed,
"documents_added": result.documents_added, 'documents_added': result.documents_added,
"errors": result.errors 'errors': result.errors,
}) }
except Exception as e: finally:
logger.error(f"reindex 异常: {e}") sync_svc.on_change_callback = old_callback
return jsonify({"error": "操作失败,请稍后重试"}), 500
else: registry.start_task(task.id, _do_reindex, sync_service, kb_name)
return jsonify({"error": "同步服务不可用"}), 503
return success_response(
data={"task_id": task.id, "message": f"重建索引任务已启动: {kb_name},通过 GET /tasks/{task.id} 查询进度"},
status_code=SYNC_SUCCESS,
message="重建索引任务已启动"
)
@kb_bp.route('/kb/route', methods=['POST']) @kb_bp.route('/kb/route', methods=['POST'])
@@ -567,7 +586,7 @@ def test_routing() -> Tuple[Any, int]:
query = data.get('query', '') query = data.get('query', '')
if not query: if not query:
return jsonify({"error": "请提供查询内容"}), 400 return error_response("MISSING_PARAMS", BAD_REQUEST, "请提供查询内容", http_status=400)
# 获取路由结果 # 获取路由结果
target_kbs = route_query( target_kbs = route_query(
@@ -579,7 +598,7 @@ def test_routing() -> Tuple[Any, int]:
# 获取意图分析 # 获取意图分析
intent = kb_router.analyze_intent(query) intent = kb_router.analyze_intent(query)
return jsonify({ return success_response(data={
"query": query, "query": query,
"user_role": user.get("role"), "user_role": user.get("role"),
"user_department": user.get("department", ""), "user_department": user.get("department", ""),
@@ -636,10 +655,10 @@ def deprecate_document(kb_name: str, filename: str) -> Tuple[Any, int]:
reason, reason,
deprecated_by=user.get('user_id', 'unknown') deprecated_by=user.get('user_id', 'unknown')
) )
return jsonify(result) return success_response(data=result)
except Exception as e: except Exception as e:
logger.error(f"操作异常: {e}") logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/restore', methods=['POST']) @kb_bp.route('/collections/<kb_name>/documents/<path:filename>/restore', methods=['POST'])
@@ -668,10 +687,10 @@ def restore_document(kb_name: str, filename: str) -> Tuple[Any, int]:
try: try:
result = kb_manager.restore_document(kb_name, filename) result = kb_manager.restore_document(kb_name, filename)
return jsonify(result) return success_response(data=result)
except Exception as e: except Exception as e:
logger.error(f"操作异常: {e}") logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/documents/<path:filename>/versions', methods=['GET']) @kb_bp.route('/collections/<kb_name>/documents/<path:filename>/versions', methods=['GET'])
@@ -719,8 +738,7 @@ def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
versions = version_query.get_document_history(kb_name, filename, limit) versions = version_query.get_document_history(kb_name, filename, limit)
versions_data = [v.to_dict() for v in versions] versions_data = [v.to_dict() for v in versions]
return jsonify({ return success_response(data={
"success": True,
"document_id": filename, "document_id": filename,
"collection": kb_name, "collection": kb_name,
"versions": versions_data, "versions": versions_data,
@@ -728,7 +746,7 @@ def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]:
}) })
except Exception as e: except Exception as e:
logger.error(f"操作异常: {e}") logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@kb_bp.route('/collections/<kb_name>/update-image-descriptions', methods=['POST']) @kb_bp.route('/collections/<kb_name>/update-image-descriptions', methods=['POST'])
@@ -760,10 +778,10 @@ def update_image_descriptions(kb_name: str) -> Tuple[Any, int]:
try: try:
result = kb_manager.update_image_descriptions(kb_name) result = kb_manager.update_image_descriptions(kb_name)
return jsonify(result) return success_response(data=result)
except Exception as e: except Exception as e:
logger.error(f"操作异常: {e}") logger.error(f"操作异常: {e}")
return jsonify({"success": False, "error": "操作失败,请稍后重试"}), 500 return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500)
@kb_bp.route('/collections/sync-vlm-cache', methods=['POST']) @kb_bp.route('/collections/sync-vlm-cache', methods=['POST'])
@@ -804,12 +822,12 @@ def sync_vlm_cache() -> Tuple[Any, int]:
images_dir = Path(".data/images") images_dir = Path(".data/images")
if not vlm_cache_dir.exists(): if not vlm_cache_dir.exists():
return jsonify({"success": False, "error": "VLM 缓存目录不存在"}), 400 return error_response("VLM_CACHE_NOT_FOUND", BAD_REQUEST, "VLM 缓存目录不存在", http_status=400)
# 获取所有 VLM 缓存文件 # 获取所有 VLM 缓存文件
cache_files = list(vlm_cache_dir.glob("*.txt")) cache_files = list(vlm_cache_dir.glob("*.txt"))
if not cache_files: if not cache_files:
return jsonify({"success": True, "total_cache_files": 0, "synced_count": 0, "message": "无 VLM 缓存文件"}) return success_response(data={"total_cache_files": 0, "synced_count": 0}, message="无 VLM 缓存文件")
# 构建图片 MD5 → 文件名 的映射 # 构建图片 MD5 → 文件名 的映射
image_hash_map = {} image_hash_map = {}
@@ -879,8 +897,7 @@ def sync_vlm_cache() -> Tuple[Any, int]:
skipped_count += 1 skipped_count += 1
details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"}) details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"})
return jsonify({ return success_response(data={
"success": True,
"total_cache_files": len(cache_files), "total_cache_files": len(cache_files),
"synced_count": synced_count, "synced_count": synced_count,
"skipped_count": skipped_count, "skipped_count": skipped_count,

View File

@@ -10,6 +10,8 @@
from flask import Blueprint, request, jsonify, current_app from flask import Blueprint, request, jsonify, current_app
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
from core.status_codes import SUCCESS, BAD_REQUEST, FORBIDDEN, NOT_FOUND, INTERNAL_ERROR, SERVICE_UNAVAILABLE, DELETE_SUCCESS
from api.response_utils import success_response, error_response
session_bp = Blueprint('session', __name__) session_bp = Blueprint('session', __name__)
@@ -34,7 +36,7 @@ def get_sessions():
""" """
session_manager = current_app.config['SESSION_MANAGER'] session_manager = current_app.config['SESSION_MANAGER']
if session_manager is None: if session_manager is None:
return jsonify({"error": "会话服务不可用"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "会话服务不可用", http_status=503)
user_id = request.current_user["user_id"] user_id = request.current_user["user_id"]
sessions = session_manager.get_user_sessions(user_id, limit=20) sessions = session_manager.get_user_sessions(user_id, limit=20)
@@ -47,7 +49,7 @@ def get_sessions():
else: else:
s["preview"] = "空会话" s["preview"] = "空会话"
return jsonify({"sessions": sessions}) return success_response(data={"sessions": sessions})
@session_bp.route('/history/<session_id>', methods=['GET']) @session_bp.route('/history/<session_id>', methods=['GET'])
@@ -65,7 +67,7 @@ def get_history(session_id):
""" """
session_manager = current_app.config['SESSION_MANAGER'] session_manager = current_app.config['SESSION_MANAGER']
if session_manager is None: if session_manager is None:
return jsonify({"error": "会话服务不可用"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "会话服务不可用", http_status=503)
user_id = request.current_user["user_id"] user_id = request.current_user["user_id"]
# 验证会话归属 # 验证会话归属
@@ -73,11 +75,11 @@ def get_history(session_id):
session_ids = [s["session_id"] for s in sessions] session_ids = [s["session_id"] for s in sessions]
if session_id not in session_ids: if session_id not in session_ids:
return jsonify({"error": "无权访问此会话"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "无权访问此会话", http_status=403)
history = session_manager.get_history(session_id, limit=100) history = session_manager.get_history(session_id, limit=100)
return jsonify({"history": history}) return success_response(data={"history": history})
@session_bp.route('/session/<session_id>', methods=['DELETE']) @session_bp.route('/session/<session_id>', methods=['DELETE'])
@@ -86,7 +88,7 @@ def delete_session(session_id):
"""删除会话""" """删除会话"""
session_manager = current_app.config['SESSION_MANAGER'] session_manager = current_app.config['SESSION_MANAGER']
if session_manager is None: if session_manager is None:
return jsonify({"error": "会话服务不可用"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "会话服务不可用", http_status=503)
user_id = request.current_user["user_id"] user_id = request.current_user["user_id"]
# 验证会话归属 # 验证会话归属
@@ -94,11 +96,11 @@ def delete_session(session_id):
session_ids = [s["session_id"] for s in sessions] session_ids = [s["session_id"] for s in sessions]
if session_id not in session_ids: if session_id not in session_ids:
return jsonify({"error": "无权删除此会话"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "无权访问此会话", http_status=403)
session_manager.delete_session(session_id) session_manager.delete_session(session_id)
return jsonify({"success": True, "message": "会话已删除"}) return success_response(status_code=DELETE_SUCCESS, message="会话已删除")
@session_bp.route('/clear/<session_id>', methods=['POST']) @session_bp.route('/clear/<session_id>', methods=['POST'])
@@ -107,7 +109,7 @@ def clear_history(session_id):
"""清空会话历史(保留会话)""" """清空会话历史(保留会话)"""
session_manager = current_app.config['SESSION_MANAGER'] session_manager = current_app.config['SESSION_MANAGER']
if session_manager is None: if session_manager is None:
return jsonify({"error": "会话服务不可用"}), 503 return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "会话服务不可用", http_status=503)
user_id = request.current_user["user_id"] user_id = request.current_user["user_id"]
# 验证会话归属 # 验证会话归属
@@ -115,8 +117,8 @@ def clear_history(session_id):
session_ids = [s["session_id"] for s in sessions] session_ids = [s["session_id"] for s in sessions]
if session_id not in session_ids: if session_id not in session_ids:
return jsonify({"error": "无权操作此会话"}), 403 return error_response("FORBIDDEN", FORBIDDEN, "无权访问此会话", http_status=403)
session_manager.clear_history(session_id) session_manager.clear_history(session_id)
return jsonify({"success": True, "message": "历史已清空"}) return success_response(message="历史已清空")

View File

@@ -31,12 +31,12 @@ Example:
""" """
from typing import Optional, Tuple, Any from typing import Optional, Tuple, Any
from flask import Blueprint, request, jsonify, current_app from flask import Blueprint, request, current_app
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from auth.gateway import require_gateway_auth from auth.gateway import require_gateway_auth
from core.status_codes import SYNC_SUCCESS, SYNC_ERROR, INTERNAL_ERROR from core.status_codes import SUCCESS, SYNC_SUCCESS, SYNC_ERROR, INTERNAL_ERROR, SERVICE_UNAVAILABLE
from api.response_utils import success_response, error_response from api.response_utils import success_response, error_response
sync_bp = Blueprint('sync', __name__) sync_bp = Blueprint('sync', __name__)
@@ -68,9 +68,7 @@ def _require_sync_service() -> Tuple[Optional[Any], Optional[Tuple]]:
service = _get_sync_service() service = _get_sync_service()
if not service: if not service:
return None, error_response( return None, error_response(
error="SERVICE_UNAVAILABLE", "SERVICE_UNAVAILABLE", INTERNAL_ERROR, "同步服务未启用",
error_code=INTERNAL_ERROR,
message="同步服务未启用",
http_status=503 http_status=503
) )
return service, None return service, None
@@ -82,9 +80,10 @@ def _require_sync_service() -> Tuple[Optional[Any], Optional[Tuple]]:
@require_gateway_auth @require_gateway_auth
def trigger_sync() -> Tuple[Any, int]: def trigger_sync() -> Tuple[Any, int]:
""" """
手动触发知识库同步 手动触发知识库同步(异步任务)
扫描文档目录,检测变更并执行向量化处理 立即返回 task_id后台线程执行同步
客户端通过 GET /tasks/<task_id> 轮询进度。
请求体 (可选): 请求体 (可选):
{ {
@@ -93,7 +92,7 @@ def trigger_sync() -> Tuple[Any, int]:
} }
Returns: Returns:
成功: {"success": true, "data": {"result": {...}}} 成功: {"success": true, "data": {"task_id": "xxx", "message": "同步任务已启动"}}
失败: {"error": "...", "error_code": "..."} 失败: {"error": "...", "error_code": "..."}
Example: Example:
@@ -104,22 +103,65 @@ def trigger_sync() -> Tuple[Any, int]:
if err: if err:
return err return err
try: from core.task_registry import get_registry
result = service.sync_now() registry = get_registry()
return success_response(
data={"result": result.to_dict() if hasattr(result, 'to_dict') else result}, # 检查是否有正在运行的同步任务
status_code=SYNC_SUCCESS, running = registry.list_tasks(status='running', task_type='sync', limit=1)
message="同步完成" if running:
)
except Exception as e:
logger.error(f"同步操作异常: {e}")
return error_response( return error_response(
error="SYNC_ERROR", "TASK_RUNNING", SYNC_ERROR,
error_code=SYNC_ERROR, f"同步任务正在执行中 (task_id: {running[0].id}),请等待完成",
message="操作失败", http_status=409
http_status=500
) )
# 创建异步任务
task = registry.create_task('sync', '文档同步')
def _do_sync(task, sync_service):
"""后台执行同步"""
# 注册进度回调
processed = [0]
def on_change(change):
processed[0] += 1
registry.update_progress(
task.id,
current=processed[0],
stage='处理文件',
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
)
old_callback = sync_service.on_change_callback
sync_service.on_change_callback = on_change
try:
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
result = sync_service.sync_now()
result_dict = result.to_dict() if hasattr(result, 'to_dict') else {
'documents_processed': result.documents_processed,
'documents_added': result.documents_added,
'documents_modified': result.documents_modified,
'documents_deleted': result.documents_deleted,
'errors': result.errors,
}
return result_dict
finally:
sync_service.on_change_callback = old_callback
registry.start_task(task.id, _do_sync, service)
return success_response(
data={
'task_id': task.id,
'message': '同步任务已启动,通过 GET /tasks/' + task.id + ' 查询进度'
},
status_code=SYNC_SUCCESS,
message="同步任务已启动"
)
@sync_bp.route('/sync/status', methods=['GET']) @sync_bp.route('/sync/status', methods=['GET'])
@require_gateway_auth @require_gateway_auth
@@ -143,12 +185,8 @@ def get_sync_status() -> Tuple[Any, int]:
""" """
service, err = _require_sync_service() service, err = _require_sync_service()
if err: if err:
return jsonify({ return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务未启用", http_status=503,
"status": "failed", enabled=False)
"status_code": INTERNAL_ERROR,
"enabled": False,
"message": "同步服务未启用"
})
try: try:
# 获取状态信息 # 获取状态信息
@@ -163,15 +201,11 @@ def get_sync_status() -> Tuple[Any, int]:
if hasattr(service, 'get_status'): if hasattr(service, 'get_status'):
status.update(service.get_status()) status.update(service.get_status())
return jsonify(status) return success_response(data=status)
except Exception as e: except Exception as e:
logger.error(f"状态查询异常: {e}") logger.error(f"状态查询异常: {e}")
return jsonify({ return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "操作失败", http_status=500,
"status": "failed", enabled=True)
"status_code": INTERNAL_ERROR,
"enabled": True,
"error": "操作失败"
})
@sync_bp.route('/sync/history', methods=['GET']) @sync_bp.route('/sync/history', methods=['GET'])
@@ -196,13 +230,11 @@ def get_sync_history() -> Tuple[Any, int]:
try: try:
history = service.get_sync_history(limit=limit) if hasattr(service, 'get_sync_history') else [] history = service.get_sync_history(limit=limit) if hasattr(service, 'get_sync_history') else []
return jsonify({"history": history}) return success_response(data={"history": history})
except Exception as e: except Exception as e:
logger.error(f"同步操作异常: {e}") logger.error(f"同步操作异常: {e}")
return error_response( return error_response(
error="SYNC_ERROR", "SYNC_ERROR", SYNC_ERROR, "操作失败",
error_code=SYNC_ERROR,
message="操作失败",
http_status=500 http_status=500
) )
@@ -231,13 +263,11 @@ def get_change_logs() -> Tuple[Any, int]:
try: try:
changes = service.get_change_logs(limit=limit, collection=collection) if hasattr(service, 'get_change_logs') else [] changes = service.get_change_logs(limit=limit, collection=collection) if hasattr(service, 'get_change_logs') else []
return jsonify({"changes": changes}) return success_response(data={"changes": changes})
except Exception as e: except Exception as e:
logger.error(f"同步操作异常: {e}") logger.error(f"同步操作异常: {e}")
return error_response( return error_response(
error="SYNC_ERROR", "SYNC_ERROR", SYNC_ERROR, "操作失败",
error_code=SYNC_ERROR,
message="操作失败",
http_status=500 http_status=500
) )
@@ -263,27 +293,23 @@ def start_sync_monitor() -> Tuple[Any, int]:
try: try:
if hasattr(service, 'is_running') and service.is_running(): if hasattr(service, 'is_running') and service.is_running():
return jsonify({"status": "success", "status_code": SYNC_SUCCESS, "message": "文件监控已在运行"}) return success_response(status_code=SYNC_SUCCESS, message="文件监控已在运行")
if hasattr(service, 'start'): if hasattr(service, 'start'):
success = service.start() success = service.start()
if success: if success:
return jsonify({"status": "success", "status_code": SYNC_SUCCESS, "message": "文件监控已启动"}) return success_response(status_code=SYNC_SUCCESS, message="文件监控已启动")
else: else:
return error_response( return error_response(
error="SYNC_ERROR", "SYNC_ERROR", SYNC_ERROR, "启动文件监控失败",
error_code=SYNC_ERROR,
message="启动文件监控失败",
http_status=500 http_status=500
) )
else: else:
return jsonify({"status": "success", "message": "文件监控功能不可用"}) return success_response(message="文件监控功能不可用")
except Exception as e: except Exception as e:
logger.error(f"同步操作异常: {e}") logger.error(f"同步操作异常: {e}")
return error_response( return error_response(
error="SYNC_ERROR", "SYNC_ERROR", SYNC_ERROR, "操作失败",
error_code=SYNC_ERROR,
message="操作失败",
http_status=500 http_status=500
) )
@@ -306,12 +332,10 @@ def stop_sync_monitor() -> Tuple[Any, int]:
try: try:
if hasattr(service, 'stop'): if hasattr(service, 'stop'):
service.stop() service.stop()
return jsonify({"status": "success", "status_code": SYNC_SUCCESS, "message": "文件监控已停止"}) return success_response(status_code=SYNC_SUCCESS, message="文件监控已停止")
except Exception as e: except Exception as e:
logger.error(f"同步操作异常: {e}") logger.error(f"同步操作异常: {e}")
return error_response( return error_response(
error="SYNC_ERROR", "SYNC_ERROR", SYNC_ERROR, "操作失败",
error_code=SYNC_ERROR,
message="操作失败",
http_status=500 http_status=500
) )

191
api/task_routes.py Normal file
View File

@@ -0,0 +1,191 @@
"""
异步任务查询 API
提供任务状态的 JSON 轮询和 SSE 流式两种查询方式。
路由列表:
GET /tasks : 任务列表
GET /tasks/<id> : 任务状态JSON
GET /tasks/<id>/progress: 任务进度SSE 流式)
GET /tasks/stats : 任务统计
后端组调用方式:
1. POST /sync → 返回 {"task_id": "xxx", ...}
2. GET /tasks/xxx → 轮询状态,直到 status 为 completed 或 failed
dev-ui 调用方式:
1. POST /sync → 返回 task_id
2. GET /tasks/xxx/progress → SSE 流式接收进度事件
"""
import json
import time
import logging
from typing import Tuple, Any
from flask import Blueprint, request, Response, stream_with_context
from auth.gateway import require_gateway_auth
from api.response_utils import success_response, error_response
from core.status_codes import SUCCESS, NOT_FOUND
logger = logging.getLogger(__name__)
task_bp = Blueprint('tasks', __name__)
def _get_registry():
"""获取任务注册表"""
from core.task_registry import get_registry
return get_registry()
# ==================== 任务查询 API ====================
@task_bp.route('/tasks', methods=['GET'])
@require_gateway_auth
def list_tasks() -> Tuple[Any, int]:
"""
获取任务列表
查询参数:
status: 过滤状态running / completed / failed / pending
type: 过滤类型sync / reindex / upload / batch_upload / exam_generate / exam_grade
limit: 返回数量限制(默认 50
Returns:
{"success": true, "data": {"tasks": [...], "total": N}}
"""
registry = _get_registry()
registry.maybe_cleanup()
status = request.args.get('status')
task_type = request.args.get('type')
limit = request.args.get('limit', 50, type=int)
tasks = registry.list_tasks(status=status, task_type=task_type, limit=limit)
return success_response(
data={
'tasks': [t.to_dict() for t in tasks],
'total': len(tasks)
},
status_code=SUCCESS,
message="查询成功"
)
@task_bp.route('/tasks/stats', methods=['GET'])
@require_gateway_auth
def task_stats() -> Tuple[Any, int]:
"""
获取任务统计信息
Returns:
{"success": true, "data": {"total": N, "by_status": {...}, "by_type": {...}}}
"""
registry = _get_registry()
return success_response(
data=registry.get_stats(),
status_code=SUCCESS,
message="查询成功"
)
@task_bp.route('/tasks/<task_id>', methods=['GET'])
@require_gateway_auth
def get_task(task_id: str) -> Tuple[Any, int]:
"""
获取单个任务状态JSON 轮询接口)
后端组推荐使用此接口轮询任务进度。
轮询建议:
- 间隔 1-2 秒
- 当 status 为 completed 或 failed 时停止轮询
Returns:
成功: {"success": true, "data": {task详情}}
未找到: {"error": "任务不存在"}
"""
registry = _get_registry()
task = registry.get_task(task_id)
if not task:
return error_response(
"TASK_NOT_FOUND", NOT_FOUND,
f"任务不存在: {task_id}",
http_status=404
)
return success_response(
data=task.to_dict(),
status_code=SUCCESS,
message="查询成功"
)
@task_bp.route('/tasks/<task_id>/progress', methods=['GET'])
@require_gateway_auth
def task_progress_stream(task_id: str):
"""
SSE 流式任务进度推送
dev-ui 前端推荐使用此接口,实时接收任务进度事件。
SSE 事件类型:
- start: 任务开始
- progress: 进度更新(含 progress/current/total/stage/message
- complete: 任务完成(含完整结果)
- error: 任务失败(含错误信息)
- heartbeat: 每 15 秒发送一次保活
Returns:
text/event-stream
"""
registry = _get_registry()
task = registry.get_task(task_id)
if not task:
return error_response(
"TASK_NOT_FOUND", NOT_FOUND,
f"任务不存在: {task_id}",
http_status=404
)
def generate_sse():
"""SSE 生成器"""
sent_complete = False
while True:
# 取出缓冲区事件
events = task.drain_events()
for event in events:
yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
if event.get('type') in ('complete', 'error'):
sent_complete = True
# 如果任务已结束且事件已发送完毕,退出
if sent_complete:
break
# 如果任务已完成但没有事件,发送最终状态后退出
if task.status in ('completed', 'failed') and not events:
final_event = {
'type': 'complete' if task.status == 'completed' else 'error',
'data': task.to_dict()
}
yield f"data: {json.dumps(final_event, ensure_ascii=False)}\n\n"
break
# 心跳保活
yield f": heartbeat\n\n"
time.sleep(1)
return Response(
stream_with_context(generate_sse()),
mimetype='text/event-stream',
headers={
'Cache-Control': 'no-cache',
'X-Accel-Buffering': 'no',
'Connection': 'keep-alive'
}
)

View File

@@ -49,6 +49,8 @@ _STATUS_MESSAGES: Dict[int, str] = {
4011: "向量库不存在", 4011: "向量库不存在",
4012: "文件内容为空", 4012: "文件内容为空",
4013: "权限不足", 4013: "权限不足",
4014: "任务不存在",
4015: "任务冲突",
# 服务端错误 (50xx) # 服务端错误 (50xx)
5000: "服务器内部错误", 5000: "服务器内部错误",
@@ -59,6 +61,7 @@ _STATUS_MESSAGES: Dict[int, str] = {
5020: "出题失败", 5020: "出题失败",
5021: "批阅失败", 5021: "批阅失败",
5030: "图片描述更新失败", 5030: "图片描述更新失败",
5040: "重建索引失败",
} }
@@ -113,6 +116,8 @@ FILE_NOT_FOUND = 4010
COLLECTION_NOT_FOUND = 4011 COLLECTION_NOT_FOUND = 4011
NO_CONTENT = 4012 NO_CONTENT = 4012
PERMISSION_DENIED = 4013 PERMISSION_DENIED = 4013
TASK_NOT_FOUND = 4014
TASK_CONFLICT = 4015
# 服务端错误 (50xx) # 服务端错误 (50xx)
INTERNAL_ERROR = 5000 INTERNAL_ERROR = 5000
@@ -123,3 +128,4 @@ SYNC_ERROR = 5010
EXAM_ERROR = 5020 EXAM_ERROR = 5020
GRADE_ERROR = 5021 GRADE_ERROR = 5021
IMAGE_DESC_ERROR = 5030 IMAGE_DESC_ERROR = 5030
REINDEX_ERROR = 5040

311
core/task_registry.py Normal file
View File

@@ -0,0 +1,311 @@
"""
异步任务注册表
为长时间运行的操作(同步、重建索引、上传向量化、出题、批阅)提供统一的
任务状态跟踪机制。支持 JSON 轮询和 SSE 流式两种进度查询方式。
核心设计:
- 进程内字典存储,线程安全
- 后台线程执行长操作,立即返回 task_id
- 任务完成后保留 1 小时自动清理
- 单 Worker 环境下足够可靠
使用示例:
from core.task_registry import get_registry
registry = get_registry()
task_id = registry.start_task('sync', '文档同步', total=10)
# 在后台线程中更新进度
registry.update_progress(task_id, current=5, message='处理中...')
registry.complete_task(task_id, result={...})
"""
import uuid
import time
import threading
import logging
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
logger = logging.getLogger(__name__)
# ==================== 数据模型 ====================
@dataclass
class TaskInfo:
"""任务信息"""
id: str # 任务唯一标识
type: str # 任务类型: sync / reindex / upload / batch_upload / exam_generate / exam_grade
description: str # 人类可读描述
status: str = 'pending' # pending / running / completed / failed
progress: float = 0.0 # 0-100 百分比
current: int = 0 # 当前处理项数
total: int = 0 # 总项数
stage: str = '' # 当前阶段描述
message: str = '' # 当前步骤消息
result: Any = None # 完成后的结果数据
error: Optional[str] = None # 失败时的错误信息
created_at: float = 0.0 # 创建时间戳
started_at: Optional[float] = None
completed_at: Optional[float] = None
_events: List[dict] = field(default_factory=list, repr=False) # SSE 事件缓冲
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
def add_event(self, event: dict):
"""添加 SSE 事件到缓冲区"""
with self._lock:
self._events.append(event)
def drain_events(self) -> List[dict]:
"""取出并清空事件缓冲区"""
with self._lock:
events = list(self._events)
self._events.clear()
return events
def to_dict(self) -> dict:
"""转为可序列化的字典"""
d = {
'task_id': self.id,
'type': self.type,
'description': self.description,
'status': self.status,
'progress': round(self.progress, 1),
'current': self.current,
'total': self.total,
'stage': self.stage,
'message': self.message,
'created_at': datetime.fromtimestamp(self.created_at).isoformat(),
}
if self.started_at:
d['started_at'] = datetime.fromtimestamp(self.started_at).isoformat()
if self.completed_at:
d['completed_at'] = datetime.fromtimestamp(self.completed_at).isoformat()
d['duration_ms'] = int((self.completed_at - self.started_at) * 1000)
if self.result is not None:
d['result'] = self.result
if self.error is not None:
d['error'] = self.error
return d
# ==================== 任务注册表 ====================
class TaskRegistry:
"""
全局任务注册表(单例)
管理所有异步任务的创建、执行、状态更新和查询。
"""
def __init__(self, max_workers: int = 4, task_ttl: int = 3600):
self._tasks: Dict[str, TaskInfo] = {}
self._lock = threading.Lock()
self._executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='task')
self._task_ttl = task_ttl # 已完成任务的保留时间(秒)
self._cleanup_interval = 300 # 清理间隔(秒)
self._last_cleanup = time.time()
def create_task(self, task_type: str, description: str, total: int = 0) -> TaskInfo:
"""
创建新任务
Args:
task_type: 任务类型标识
description: 人类可读描述
total: 预计处理的总项数(用于计算百分比)
Returns:
TaskInfo 实例
"""
task_id = uuid.uuid4().hex[:12]
task = TaskInfo(
id=task_id,
type=task_type,
description=description,
status='pending',
total=total,
created_at=time.time(),
)
with self._lock:
self._tasks[task_id] = task
logger.info(f"[任务] 创建 {task_id}: {description}")
return task
def start_task(self, task_id: str, fn: Callable, *args, **kwargs) -> str:
"""
在后台线程中执行任务
Args:
task_id: 任务 ID
fn: 要执行的函数,签名为 fn(task, *args, **kwargs)
*args, **kwargs: 传给 fn 的参数
Returns:
task_id
"""
task = self._tasks.get(task_id)
if not task:
raise ValueError(f"任务不存在: {task_id}")
self._submit(fn, task, args, kwargs)
return task_id
def _submit(self, fn, task, args, kwargs):
"""提交任务到线程池"""
def _wrapper():
task.status = 'running'
task.started_at = time.time()
task.add_event({'type': 'start', 'data': {'stage': task.stage or '初始化'}})
try:
result = fn(task, *args, **kwargs)
task.status = 'completed'
task.progress = 100.0
task.completed_at = time.time()
task.result = result
task.add_event({'type': 'complete', 'data': task.to_dict()})
logger.info(f"[任务] 完成 {task.id}: {task.description}")
except Exception as e:
task.status = 'failed'
task.error = str(e)
task.completed_at = time.time()
task.add_event({'type': 'error', 'data': {'message': str(e)}})
logger.error(f"[任务] 失败 {task.id}: {e}", exc_info=True)
self._executor.submit(_wrapper)
def get_task(self, task_id: str) -> Optional[TaskInfo]:
"""获取任务信息"""
with self._lock:
return self._tasks.get(task_id)
def list_tasks(self, status: Optional[str] = None, task_type: Optional[str] = None,
limit: int = 50) -> List[TaskInfo]:
"""列出任务(按创建时间倒序)"""
with self._lock:
tasks = list(self._tasks.values())
if status:
tasks = [t for t in tasks if t.status == status]
if task_type:
tasks = [t for t in tasks if t.type == task_type]
tasks.sort(key=lambda t: t.created_at, reverse=True)
return tasks[:limit]
def update_progress(self, task_id: str, *, current: Optional[int] = None,
total: Optional[int] = None, stage: Optional[str] = None,
message: Optional[str] = None):
"""
更新任务进度(线程安全)
Args:
task_id: 任务 ID
current: 当前处理项数
total: 更新总项数
stage: 当前阶段
message: 当前步骤描述
"""
task = self._tasks.get(task_id)
if not task:
return
with task._lock:
if current is not None:
task.current = current
if total is not None:
task.total = total
if stage is not None:
task.stage = stage
if message is not None:
task.message = message
# 计算百分比
if task.total > 0:
task.progress = min(99.0, (task.current / task.total) * 100)
# 推送进度事件
task.add_event({
'type': 'progress',
'data': {
'progress': round(task.progress, 1),
'current': task.current,
'total': task.total,
'stage': task.stage,
'message': task.message,
}
})
def complete_task(self, task_id: str, result: Any = None):
"""手动标记任务完成"""
task = self._tasks.get(task_id)
if not task:
return
task.status = 'completed'
task.progress = 100.0
task.completed_at = time.time()
if result is not None:
task.result = result
task.add_event({'type': 'complete', 'data': task.to_dict()})
def fail_task(self, task_id: str, error: str):
"""手动标记任务失败"""
task = self._tasks.get(task_id)
if not task:
return
task.status = 'failed'
task.error = error
task.completed_at = time.time()
task.add_event({'type': 'error', 'data': {'message': error}})
def cleanup(self):
"""清理过期的已完成任务"""
now = time.time()
with self._lock:
expired = [
tid for tid, t in self._tasks.items()
if t.status in ('completed', 'failed')
and t.completed_at
and (now - t.completed_at) > self._task_ttl
]
for tid in expired:
del self._tasks[tid]
if expired:
logger.debug(f"[任务] 清理 {len(expired)} 个过期任务")
def maybe_cleanup(self):
"""定期清理(避免频繁操作)"""
if time.time() - self._last_cleanup > self._cleanup_interval:
self.cleanup()
self._last_cleanup = time.time()
def get_stats(self) -> dict:
"""获取任务统计"""
with self._lock:
tasks = list(self._tasks.values())
stats = {'total': len(tasks), 'by_status': {}, 'by_type': {}}
for t in tasks:
stats['by_status'][t.status] = stats['by_status'].get(t.status, 0) + 1
stats['by_type'][t.type] = stats['by_type'].get(t.type, 0) + 1
return stats
# ==================== 全局单例 ====================
_registry: Optional[TaskRegistry] = None
_registry_lock = threading.Lock()
def get_registry() -> TaskRegistry:
"""获取全局任务注册表单例"""
global _registry
if _registry is None:
with _registry_lock:
if _registry is None:
_registry = TaskRegistry()
return _registry

View File

@@ -22,6 +22,7 @@
- [12. 图片服务](#12-图片服务) - [12. 图片服务](#12-图片服务)
- [13. 报告服务](#13-报告服务) - [13. 报告服务](#13-报告服务)
- [14. 知识库路由](#14-知识库路由) - [14. 知识库路由](#14-知识库路由)
- [15. 异步任务查询](#15-异步任务查询)
- [附录. 已知问题](#附录-已知问题) - [附录. 已知问题](#附录-已知问题)
--- ---
@@ -587,15 +588,18 @@ curl -s -X POST http://localhost:5001/documents/upload \
"size": 18, "size": 18,
"replaced": false "replaced": false
}, },
"sync_status": "已保存并添加到向量库" "sync_status": "已保存,向量化任务已启动",
"task_id": "a1b2c3d4e5f6"
}, },
"message": "文件上传成功,已保存并添加到向量库", "message": "文件上传成功,已保存,向量化任务已启动",
"status": "success", "status": "success",
"status_code": 2002, "status_code": 2002,
"success": true "success": true
} }
``` ```
> **异步说明**:文件保存为同步操作,向量化在后台线程异步执行。响应中的 `task_id` 可用于轮询向量化进度(`GET /tasks/<task_id>`)。
>
> **同名文件处理**:上传同名文件时,旧版本的切片会被自动清理后覆盖(`replaced: true`),不会生成时间戳后缀文件。 > **同名文件处理**:上传同名文件时,旧版本的切片会被自动清理后覆盖(`replaced: true`),不会生成时间戳后缀文件。
**验证结果**:✅ 通过 **验证结果**:✅ 通过
@@ -624,7 +628,8 @@ curl -s -X POST http://localhost:5001/documents/batch-upload \
{"filename": "file2.txt", "path": "public_kb/file2.txt", "status": "success", "replaced": false} {"filename": "file2.txt", "path": "public_kb/file2.txt", "status": "success", "replaced": false}
], ],
"success_count": 2, "success_count": 2,
"total": 2 "total": 2,
"task_id": "b2c3d4e5f6a1"
}, },
"message": "批量上传完成,成功 2/2 个文件", "message": "批量上传完成,成功 2/2 个文件",
"status": "success", "status": "success",
@@ -633,6 +638,8 @@ curl -s -X POST http://localhost:5001/documents/batch-upload \
} }
``` ```
> **异步说明**:批量上传成功后自动触发后台向量化任务。`task_id` 可用于轮询向量化进度(`GET /tasks/<task_id>`)。若无成功上传的文件则不返回 `task_id`。
>
> **同名文件处理**:与单文件上传相同,批量上传中遇到同名文件也会自动覆盖旧版本(`replaced: true`)。 > **同名文件处理**:与单文件上传相同,批量上传中遇到同名文件也会自动覆盖旧版本(`replaced: true`)。
**验证结果**:✅ 通过 **验证结果**:✅ 通过
@@ -882,7 +889,7 @@ curl -s -X DELETE "http://localhost:5001/chunks/人员名册.txt_text_0?collecti
### POST /sync ### POST /sync
触发文档同步。 触发文档同步(异步任务)
```bash ```bash
curl -s -X POST http://localhost:5001/sync \ curl -s -X POST http://localhost:5001/sync \
@@ -893,24 +900,23 @@ curl -s -X POST http://localhost:5001/sync \
```json ```json
{ {
"data": { "data": {
"result": { "task_id": "c3d4e5f6a1b2",
"documents_added": 1, "message": "同步任务已启动,通过 GET /tasks/c3d4e5f6a1b2 查询进度"
"documents_deleted": 1,
"documents_modified": 0,
"documents_processed": 2,
"end_time": "2026-05-04T01:45:42",
"errors": [],
"start_time": "2026-05-04T01:45:41",
"status": "completed"
}
}, },
"message": "同步完成", "message": "同步任务已启动",
"status": "success", "status": "success",
"status_code": 2010, "status_code": 2010,
"success": true "success": true
} }
``` ```
> **⚠️ 异步变更**:此接口已从同步改为异步。不再直接返回同步结果,而是返回 `task_id`。后端需通过 `GET /tasks/<task_id>` 轮询任务状态,直到 `status` 为 `completed` 或 `failed`。
>
> **冲突检测**:如果已有同步任务正在运行,返回 HTTP 409
> ```json
> {"error": "TASK_RUNNING", "message": "同步任务正在执行中 (task_id: xxx),请等待完成"}
> ```
**验证结果**:✅ 通过 **验证结果**:✅ 通过
--- ---
@@ -1456,44 +1462,54 @@ curl -s -X POST http://localhost:5001/exam/generate \
```json ```json
{ {
"data": { "data": {
"questions": [ "task_id": "d4e5f6a1b2c3",
{ "message": "出题任务已启动 (10题),通过 GET /tasks/d4e5f6a1b2c3 查询结果"
"content": {
"stem": "重购率的定义是下列哪一项?",
"answer": "B",
"data": {
"options": [
{"content": "选项A内容", "key": "A"},
{"content": "品规连续两周订货客户数/上周订货客户数*100%", "key": "B"}
]
},
"explanation": "重购率的定义是..."
},
"difficulty": 3,
"question_type": "single_choice",
"source_trace": {
"chunk_ids": ["1.docx_76"],
"document_name": "public_kb/1.docx",
"page_numbers": [1],
"sources": [...]
}
}
],
"request_id": null,
"source_chunks_used": 15,
"success": true,
"total": 10,
"requested_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"actual_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"warnings": []
}, },
"message": "出题成功", "message": "出题任务已启动",
"status": "success", "status": "success",
"status_code": 2020, "status_code": 2020,
"success": true "success": true
} }
``` ```
> **⚠️ 异步变更**:此接口已从同步改为异步。响应仅返回 `task_id`,后端需通过 `GET /tasks/<task_id>` 轮询任务状态。任务完成后,`result` 字段包含完整出题结果(格式见下方说明)。
**轮询结果GET /tasks/\<task_id\> 完成后的 result 字段)**
```json
{
"questions": [
{
"content": {
"stem": "重购率的定义是下列哪一项?",
"answer": "B",
"data": {
"options": [
{"content": "选项A内容", "key": "A"},
{"content": "品规连续两周订货客户数/上周订货客户数*100%", "key": "B"}
]
},
"explanation": "重购率的定义是..."
},
"difficulty": 3,
"question_type": "single_choice",
"source_trace": {
"chunk_ids": ["1.docx_76"],
"document_name": "public_kb/1.docx",
"page_numbers": [1],
"sources": [...]
}
}
],
"request_id": null,
"source_chunks_used": 15,
"success": true,
"total": 10,
"requested_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"actual_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"warnings": []
}
```
> **说明**`warnings` 字段在某题型实际生成数量少于请求数量时返回提示信息。 > **说明**`warnings` 字段在某题型实际生成数量少于请求数量时返回提示信息。
**验证结果**:✅ 通过2026-06-05 **验证结果**:✅ 通过2026-06-05
@@ -1527,31 +1543,41 @@ curl -s -X POST http://localhost:5001/exam/generate-smart \
```json ```json
{ {
"data": { "data": {
"ai_analysis": { "task_id": "e5f6a1b2c3d4",
"total_knowledge_points": 21, "message": "AI 智能出题任务已启动,通过 GET /tasks/e5f6a1b2c3d4 查询结果"
"suitable_types": ["single_choice", "multiple_choice", "true_false", "subjective"],
"question_types": {
"single_choice": 8,
"multiple_choice": 6,
"true_false": 4,
"fill_blank": 0,
"subjective": 3
},
"reason": "文档包含21个知识点涵盖术语定义、流程步骤、数值标准..."
},
"questions": [...],
"request_id": null,
"source_chunks_used": 15,
"success": true,
"total": 16
}, },
"message": "AI 智能出题成功", "message": "AI 智能出题任务已启动",
"status": "success", "status": "success",
"status_code": 2020, "status_code": 2020,
"success": true "success": true
} }
``` ```
> **⚠️ 异步变更**:此接口已从同步改为异步。响应仅返回 `task_id`,后端需通过 `GET /tasks/<task_id>` 轮询任务状态。任务完成后,`result` 字段包含完整出题结果(含 `ai_analysis` 字段)。
**轮询结果GET /tasks/\<task_id\> 完成后的 result 字段)**
```json
{
"ai_analysis": {
"total_knowledge_points": 21,
"suitable_types": ["single_choice", "multiple_choice", "true_false", "subjective"],
"question_types": {
"single_choice": 8,
"multiple_choice": 6,
"true_false": 4,
"fill_blank": 0,
"subjective": 3
},
"reason": "文档包含21个知识点涵盖术语定义、流程步骤、数值标准..."
},
"questions": [...],
"request_id": null,
"source_chunks_used": 15,
"success": true,
"total": 16
}
```
**注意事项** **注意事项**
- 实际出题数量 ≤ min(文档知识点数, AI 推荐数量) - 实际出题数量 ≤ min(文档知识点数, AI 推荐数量)
- 如果文档知识点较少,生成的题目数量会相应减少 - 如果文档知识点较少,生成的题目数量会相应减少
@@ -1601,33 +1627,43 @@ curl -s -X POST http://localhost:5001/exam/grade \
```json ```json
{ {
"data": { "data": {
"request_id": null, "task_id": "f6a1b2c3d4e5",
"results": [ "message": "批阅任务已启动 (1题),通过 GET /tasks/f6a1b2c3d4e5 查询结果"
{
"question_id": "q1",
"score": 2,
"max_score": 2,
"grading_status": "success",
"details": {
"correct": true,
"student_answer": "B",
"correct_answer": "B",
"feedback": "正确!"
}
}
],
"score_rate": 100.0,
"success": true,
"total_max_score": 2,
"total_score": 2
}, },
"message": "批阅完成", "message": "批阅任务已启动",
"status": "success", "status": "success",
"status_code": 2021, "status_code": 2021,
"success": true "success": true
} }
``` ```
> **⚠️ 异步变更**:此接口已从同步改为异步。响应仅返回 `task_id`,后端需通过 `GET /tasks/<task_id>` 轮询任务状态。任务完成后,`result` 字段包含完整批阅结果(格式见下方说明)。
**轮询结果GET /tasks/\<task_id\> 完成后的 result 字段)**
```json
{
"request_id": null,
"results": [
{
"question_id": "q1",
"score": 2,
"max_score": 2,
"grading_status": "success",
"details": {
"correct": true,
"student_answer": "B",
"correct_answer": "B",
"feedback": "正确!"
}
}
],
"score_rate": 100.0,
"success": true,
"total_max_score": 2,
"total_score": 2
}
```
**`grading_status` 取值说明** **`grading_status` 取值说明**
- `success`:评分成功 - `success`:评分成功
- `failed`:评分失败(主观题 LLM 解析失败或超时),此时 `details` 包含 `error` 字段 - `failed`:评分失败(主观题 LLM 解析失败或超时),此时 `details` 包含 `error` 字段
@@ -1823,6 +1859,190 @@ curl -s -X POST http://localhost:5001/kb/route \
--- ---
## 15. 异步任务查询
> 所有异步操作(同步、重建索引、上传向量化、出题、批阅)返回的 `task_id` 均可通过以下接口查询进度。
### GET /tasks
获取任务列表。
```bash
curl -s http://localhost:5001/tasks
```
**查询参数**
| 参数 | 类型 | 必需 | 说明 |
|------|------|------|------|
| `status` | string | ❌ | 过滤状态:`pending` / `running` / `completed` / `failed` |
| `type` | string | ❌ | 过滤类型:`sync` / `reindex` / `upload` / `batch_upload` / `exam_generate` / `exam_grade` |
| `limit` | int | ❌ | 返回数量限制(默认 50 |
**响应示例**
```json
{
"data": {
"tasks": [
{
"task_id": "a1b2c3d4e5f6",
"type": "sync",
"description": "文档同步",
"status": "running",
"progress": 45.0,
"current": 9,
"total": 20,
"stage": "处理文件",
"message": "已处理: 产品手册.pdf",
"created_at": "2026-06-05T10:30:00"
}
],
"total": 1
},
"message": "查询成功",
"status": "success",
"status_code": 2000,
"success": true
}
```
**验证结果**:✅ 通过
---
### GET /tasks/\<task_id\>
获取单个任务状态JSON 轮询接口)。
**后端组推荐使用此接口轮询任务进度,建议间隔 1-2 秒。**
```bash
curl -s http://localhost:5001/tasks/a1b2c3d4e5f6
```
**响应示例(运行中)**
```json
{
"data": {
"task_id": "a1b2c3d4e5f6",
"type": "sync",
"description": "文档同步",
"status": "running",
"progress": 45.0,
"current": 9,
"total": 20,
"stage": "处理文件",
"message": "已处理: 产品手册.pdf",
"created_at": "2026-06-05T10:30:00",
"started_at": "2026-06-05T10:30:01"
},
"message": "查询成功",
"status": "success",
"status_code": 2000,
"success": true
}
```
**响应示例(已完成)**
```json
{
"data": {
"task_id": "a1b2c3d4e5f6",
"type": "sync",
"description": "文档同步",
"status": "completed",
"progress": 100.0,
"current": 20,
"total": 20,
"stage": "完成",
"message": "同步完成",
"created_at": "2026-06-05T10:30:00",
"started_at": "2026-06-05T10:30:01",
"completed_at": "2026-06-05T10:30:15",
"duration_ms": 14000,
"result": {
"documents_processed": 20,
"documents_added": 3,
"documents_modified": 2,
"documents_deleted": 0,
"errors": []
}
},
"message": "查询成功",
"status": "success",
"status_code": 2000,
"success": true
}
```
> **任务状态**
> - `pending`:已创建,等待执行
> - `running`:正在执行
> - `completed`:执行完成,`result` 字段包含完整结果
> - `failed`:执行失败,`error` 字段包含错误信息
>
> **轮询建议**:间隔 1-2 秒,当 `status` 为 `completed` 或 `failed` 时停止轮询。
**验证结果**:✅ 通过
---
### GET /tasks/\<task_id\>/progress
SSE 流式任务进度推送dev-ui 前端推荐使用)。
```bash
curl -s -N http://localhost:5001/tasks/a1b2c3d4e5f6/progress
```
**SSE 事件序列**
```
data: {"type": "start", "data": {"stage": "扫描文档"}}
data: {"type": "progress", "data": {"progress": 10.0, "current": 2, "total": 20, "stage": "处理文件", "message": "已处理: file1.pdf"}}
data: {"type": "progress", "data": {"progress": 25.0, "current": 5, "total": 20, "stage": "处理文件", "message": "已处理: file2.docx"}}
data: {"type": "complete", "data": {"task_id": "a1b2c3d4e5f6", "status": "completed", "result": {...}}}
```
> **SSE 事件类型**
> - `start`:任务开始
> - `progress`:进度更新(包含 progress/current/total/stage/message
> - `complete`任务完成data 为完整任务详情,含 result
> - `error`任务失败data 包含 message 错误信息)
> - 每 1 秒发送一次 `: heartbeat` 保活
**验证结果**:✅ 通过
---
### GET /tasks/stats
获取任务统计信息。
```bash
curl -s http://localhost:5001/tasks/stats
```
**响应示例**
```json
{
"data": {
"total": 5,
"by_status": {"running": 1, "completed": 3, "failed": 1},
"by_type": {"sync": 2, "exam_generate": 2, "upload": 1}
},
"message": "查询成功",
"status": "success",
"status_code": 2000,
"success": true
}
```
**验证结果**:✅ 通过
---
## 附录. 已知问题 ## 附录. 已知问题
### 1. /rag 接口 collections 参数 ### 1. /rag 接口 collections 参数
@@ -1951,11 +2171,11 @@ curl -s -X POST http://localhost:5001/kb/route \
当代码升级涉及元数据字段变更时(如新增 `chunk_index``doc_type`),需要重构向量库: 当代码升级涉及元数据字段变更时(如新增 `chunk_index``doc_type`),需要重构向量库:
```bash ```bash
# 重构指定知识库(清除哈希记录 → 触发全量同步) # 重构指定知识库(清除哈希记录 → 触发全量同步,异步任务
curl -s -X POST http://127.0.0.1:5001/collections/<kb_name>/reindex curl -s -X POST http://127.0.0.1:5001/collections/<kb_name>/reindex
``` ```
> **⚠️ 注意**reindex 会调用 `sync_now()` 全局同步,期间 gunicorn worker 被阻塞,搜索和问答接口将暂时无响应。建议在低峰期执行 > **⚠️ 注意**reindex 已改为异步任务,立即返回 `task_id`。通过 `GET /tasks/<task_id>` 轮询进度。不再阻塞 gunicorn worker搜索和问答接口不受影响
### Reranker 配置 ### Reranker 配置

View File

@@ -4,9 +4,33 @@
## 📋 变更记录2026-06-05 更新) ## 📋 变更记录2026-06-05 更新)
> **本次更新内容**:新增 AI 智能出题端口、更新生产环境测试结果 > **本次更新内容**:新增 AI 智能出题端口、更新生产环境测试结果、**长操作改为异步任务**
> >
> **2026-06-05 更新**:出题批卷接口格式优化与输入校验增强 > **2026-06-05 更新**:出题批卷接口格式优化与输入校验增强
>
> **2026-06-05 异步任务变更**:同步、上传向量化、出题、批阅等长耗时操作改为异步任务模式,立即返回 `task_id`,通过 `GET /tasks/<task_id>` 轮询结果
### 异步任务变更(⚠️ 重要2026-06-05
| 端点 | 变更说明 |
|------|----------|
| `POST /sync` | 改为异步任务,返回 `{"task_id": "xxx"}` 而非同步结果 |
| `POST /documents/sync` | 改为异步任务,返回 `{"task_id": "xxx"}` |
| `POST /collections/<kb>/reindex` | 改为异步任务,返回 `{"task_id": "xxx"}` |
| `POST /documents/upload` | 新增 `task_id` 字段(向量化后台执行) |
| `POST /documents/batch-upload` | 新增 `task_id` 字段(批量向量化后台执行) |
| `POST /exam/generate` | 改为异步任务,返回 `{"task_id": "xxx"}` |
| `POST /exam/generate-smart` | 改为异步任务,返回 `{"task_id": "xxx"}` |
| `POST /exam/grade` | 改为异步任务,返回 `{"task_id": "xxx"}` |
**新增任务查询接口**
| 端点 | 方法 | 功能 | 说明 |
|------|------|------|------|
| `/tasks` | GET | 任务列表 | 支持按 status/type 过滤 |
| `/tasks/<task_id>` | GET | 任务状态JSON | 后端组推荐轮询接口,建议 1-2 秒间隔 |
| `/tasks/<task_id>/progress` | GET | 任务进度SSE | dev-ui 前端推荐使用 |
| `/tasks/stats` | GET | 任务统计 | 按状态和类型分组统计 |
### 新增端口 ### 新增端口
@@ -123,10 +147,22 @@ RAG服务负责
### 2.4 出题系统(可选) ### 2.4 出题系统(可选)
| 端点 | 方法 | 功能 | | 端点 | 方法 | 功能 | 说明 |
|-----|------|------| |-----|------|------|------|
| `/exam/generate` | POST | 生成题目 | | `/exam/generate` | POST | 生成题目 | 异步任务,返回 task_id |
| `/exam/grade` | POST | 批阅答案 | | `/exam/generate-smart` | POST | AI 智能出题 | 异步任务,返回 task_id |
| `/exam/grade` | POST | 批阅答案 | 异步任务,返回 task_id |
### 2.5 异步任务查询
> 所有异步操作(同步、重建索引、上传向量化、出题、批阅)返回的 `task_id` 均可通过以下接口查询进度。
| 端点 | 方法 | 功能 | 说明 |
|-----|------|------|------|
| `/tasks` | GET | 任务列表 | 支持按 status/type 过滤 |
| `/tasks/<task_id>` | GET | 任务状态JSON | **后端组推荐轮询接口**,建议 1-2 秒间隔 |
| `/tasks/<task_id>/progress` | GET | 任务进度SSE | dev-ui 前端推荐使用 |
| `/tasks/stats` | GET | 任务统计 | 按状态和类型分组统计 |
--- ---
@@ -247,7 +283,7 @@ ENABLE_DIFY_WORKFLOW=false
--- ---
**最后更新**: 2026-04-29 **最后更新**: 2026-06-05
> 本文档供后端开发人员参考,用于对接 RAG 知识库服务。 > 本文档供后端开发人员参考,用于对接 RAG 知识库服务。
@@ -1089,17 +1125,24 @@ Content-Type: multipart/form-data
```json ```json
{ {
"success": true, "success": true,
"message": "文件上传成功,已保存并添加到向量库", "status_code": 2002,
"file": { "message": "文件上传成功,已保存,向量化任务已启动",
"filename": "document.pdf", "data": {
"collection": "public_kb", "file": {
"path": "public_kb/document.pdf", "filename": "document.pdf",
"size": 1024000, "collection": "public_kb",
"replaced": false "path": "public_kb/document.pdf",
"size": 1024000,
"replaced": false
},
"sync_status": "已保存,向量化任务已启动",
"task_id": "a1b2c3d4e5f6"
} }
} }
``` ```
> **异步说明**:文件保存为同步操作,向量化在后台线程异步执行。响应中的 `task_id` 可用于轮询向量化进度(`GET /tasks/<task_id>`)。当同步服务不可用时,`task_id` 为 `null``sync_status` 为 `"已保存,等待手动同步"`。
**同名文件处理**:上传同名文件时,旧版本的切片会被自动清理后覆盖(`replaced: true`),不会生成时间戳后缀文件。这确保了向量库中不会出现同一文档的新旧切片共存的情况。 **同名文件处理**:上传同名文件时,旧版本的切片会被自动清理后覆盖(`replaced: true`),不会生成时间戳后缀文件。这确保了向量库中不会出现同一文档的新旧切片共存的情况。
### 5.2 批量上传 ### 5.2 批量上传
@@ -1257,26 +1300,51 @@ POST /sync
**请求体:** 无需传递参数(同步所有知识库) **请求体:** 无需传递参数(同步所有知识库)
**响应:** **响应(异步任务)**
```json ```json
{ {
"success": true, "success": true,
"status": "success",
"status_code": 2010, "status_code": 2010,
"message": "同步完成", "message": "同步任务已启动",
"data": { "data": {
"result": { "task_id": "c3d4e5f6a1b2",
"documents_added": 1, "message": "同步任务已启动,通过 GET /tasks/c3d4e5f6a1b2 查询进度"
"documents_deleted": 1,
"documents_modified": 0,
"documents_processed": 2,
"errors": [],
"status": "completed"
}
} }
} }
``` ```
> **⚠️ 异步变更**:此接口已从同步改为异步。不再直接返回同步结果,而是返回 `task_id`。后端需通过 `GET /tasks/<task_id>` 轮询任务状态,直到 `status` 为 `completed` 或 `failed`。任务完成后,`result` 字段包含完整的同步结果(含 `documents_processed`、`documents_added` 等)。
>
> **冲突检测**:如果已有同步任务正在运行,返回 HTTP 409`{"error": "TASK_RUNNING", "message": "同步任务正在执行中 (task_id: xxx),请等待完成"}`
**后端轮询示例**
```python
import time
import requests
def trigger_sync_and_wait():
"""触发同步并等待完成"""
# 1. 触发同步任务
resp = requests.post('http://rag-service:5001/sync')
task_id = resp.json()['data']['task_id']
# 2. 轮询任务状态(每 2 秒)
while True:
time.sleep(2)
status_resp = requests.get(f'http://rag-service:5001/tasks/{task_id}')
task_data = status_resp.json()['data']
if task_data['status'] == 'completed':
print(f"同步完成: {task_data['result']}")
return task_data['result']
elif task_data['status'] == 'failed':
raise Exception(f"同步失败: {task_data['error']}")
else:
print(f"同步中: {task_data['progress']}% - {task_data['message']}")
```
```
### 6.2 同步状态 ### 6.2 同步状态
``` ```
@@ -1359,7 +1427,7 @@ POST /sync/stop
```json ```json
{ {
"status": "success", "status": "success",
"status_code": 3001, "status_code": 2010,
"message": "文件监控已启动" "message": "文件监控已启动"
} }
``` ```
@@ -1417,26 +1485,24 @@ POST /exam/generate
| difficulty | 必须为 1-5 的整数 | HTTP 400 INVALID_PARAMS | | difficulty | 必须为 1-5 的整数 | HTTP 400 INVALID_PARAMS |
| **总题数上限** | **所有题型数量之和不能超过 20** | HTTP 400 INVALID_PARAMS | | **总题数上限** | **所有题型数量之和不能超过 20** | HTTP 400 INVALID_PARAMS |
**响应:** **响应(异步任务)**
**完整响应格式**(包含外层包装):
```json ```json
{ {
"success": true, "success": true,
"status": "success", "status_code": 2020,
"status_code": 2011, "message": "出题任务已启动",
"message": "出题完成",
"data": { "data": {
"success": true, "task_id": "d4e5f6a1b2c3",
"request_id": "xxx", "message": "出题任务已启动 (10题),通过 GET /tasks/d4e5f6a1b2c3 查询结果"
"total": 10,
"source_chunks_used": 15,
"questions": [...]
} }
} }
``` ```
**data 内部结构** > **⚠️ 异步变更**:此接口已从同步改为异步。响应仅返回 `task_id`,后端需通过 `GET /tasks/<task_id>` 轮询任务状态。任务完成后,`result` 字段包含完整出题结果(格式见下方说明)。
**轮询结果GET /tasks/\<task_id\> 完成后的 result 字段)**
```json ```json
{ {
"success": true, "success": true,
@@ -1635,25 +1701,22 @@ POST /exam/grade
#### 响应 #### 响应
**完整响应格式**(包含外层包装): **响应(异步任务):**
```json ```json
{ {
"success": true, "success": true,
"status": "success",
"status_code": 2021, "status_code": 2021,
"message": "批阅完成", "message": "批阅任务已启动",
"data": { "data": {
"request_id": "可选,原样返回", "task_id": "f6a1b2c3d4e5",
"success": true, "message": "批阅任务已启动 (5题),通过 GET /tasks/f6a1b2c3d4e5 查询结果"
"total_score": 12.5,
"total_max_score": 22.0,
"score_rate": 56.8,
"results": [...]
} }
} }
``` ```
**data 内部结构** > **⚠️ 异步变更**:此接口已从同步改为异步。响应仅返回 `task_id`,后端需通过 `GET /tasks/<task_id>` 轮询任务状态。任务完成后,`result` 字段包含完整批阅结果(格式见下方说明)。
**轮询结果GET /tasks/\<task_id\> 完成后的 result 字段)**
```json ```json
{ {
@@ -1817,16 +1880,30 @@ def build_grade_request(answer_list, questions_map):
return {"answers": grade_answers} return {"answers": grade_answers}
``` ```
**Step 4: 调用 RAG 批卷接口** **Step 4: 调用 RAG 批卷接口(异步任务)**
```python ```python
import time
def call_rag_grade(grade_request): def call_rag_grade(grade_request):
"""调用 RAG 批卷接口""" """调用 RAG 批卷接口并轮询等待结果"""
# 1. 提交批阅任务
response = requests.post( response = requests.post(
'http://rag-service:5001/exam/grade', 'http://rag-service:5001/exam/grade',
json=grade_request json=grade_request
) )
return response.json() task_id = response.json()['data']['task_id']
# 2. 轮询任务状态(每 2 秒)
while True:
time.sleep(2)
status_resp = requests.get(f'http://rag-service:5001/tasks/{task_id}')
task_data = status_resp.json()['data']
if task_data['status'] == 'completed':
return task_data['result']
elif task_data['status'] == 'failed':
raise Exception(f"批阅失败: {task_data['error']}")
``` ```
**Step 5: 更新学生成绩** **Step 5: 更新学生成绩**
@@ -2430,19 +2507,22 @@ ChromaDB 集合名称限制:
| 端点 | 方法 | 说明 | | 端点 | 方法 | 说明 |
|------|------|------| |------|------|------|
| `/sync` | POST | 触发同步 | | `/sync` | POST | 触发同步**异步任务,返回 task_id** |
| `/sync/status` | GET | 同步状态 | | `/sync/status` | GET | 同步状态 |
| `/sync/history` | GET | 同步历史 | | `/sync/history` | GET | 同步历史 |
| `/sync/changes` | GET | 变更日志 | | `/sync/changes` | GET | 变更日志 |
| `/sync/start` | POST | 启动文件监控 | | `/sync/start` | POST | 启动文件监控 |
| `/sync/stop` | POST | 停止文件监控 | | `/sync/stop` | POST | 停止文件监控 |
| `/documents/sync` | POST | 触发文档同步(**异步任务,返回 task_id** |
| `/collections/<kb_name>/reindex` | POST | 重建索引(**异步任务,返回 task_id** |
### 13.7 出题系统 ### 13.7 出题系统
| 端点 | 方法 | 说明 | | 端点 | 方法 | 说明 |
|------|------|------| |------|------|------|
| `/exam/generate` | POST | 生成题目 | | `/exam/generate` | POST | 生成题目**异步任务,返回 task_id** |
| `/exam/grade` | POST | 批改答案 | | `/exam/generate-smart` | POST | AI 智能出题(**异步任务,返回 task_id** |
| `/exam/grade` | POST | 批改答案(**异步任务,返回 task_id** |
| `/exam/health` | GET | 出题服务健康检查 | | `/exam/health` | GET | 出题服务健康检查 |
### 13.8 反馈与 FAQ 管理 ### 13.8 反馈与 FAQ 管理
@@ -2497,6 +2577,77 @@ ChromaDB 集合名称限制:
|------|------|------| |------|------|------|
| `/documents/<path>/preview` | GET | 文档预览,按 `chunk_index` 跳转到具体切片dev-ui 引用溯源用) | | `/documents/<path>/preview` | GET | 文档预览,按 `chunk_index` 跳转到具体切片dev-ui 引用溯源用) |
### 13.13 异步任务查询
> 所有异步操作(同步、重建索引、上传向量化、出题、批阅)返回的 `task_id` 均可通过以下接口查询进度。
| 端点 | 方法 | 说明 |
|------|------|------|
| `/tasks` | GET | 任务列表(支持 status/type 过滤) |
| `/tasks/<task_id>` | GET | 任务状态JSON 轮询,**后端组推荐使用** |
| `/tasks/<task_id>/progress` | GET | 任务进度SSE 流式dev-ui 前端使用) |
| `/tasks/stats` | GET | 任务统计 |
**任务状态字段说明**
| 字段 | 类型 | 说明 |
|------|------|------|
| `task_id` | string | 任务唯一标识 |
| `type` | string | 类型sync / reindex / upload / batch_upload / exam_generate / exam_grade |
| `status` | string | 状态pending / running / completed / failed |
| `progress` | float | 进度百分比0-100 |
| `current` | int | 当前处理项数 |
| `total` | int | 总项数 |
| `stage` | string | 当前阶段 |
| `message` | string | 当前步骤描述 |
| `result` | any | 完成后的结果数据(仅 status=completed 时存在) |
| `error` | string | 失败错误信息(仅 status=failed 时存在) |
| `duration_ms` | int | 执行耗时毫秒(仅已完成时存在) |
**后端对接轮询模式**
```python
import time
import requests
def async_task_poll(task_id, base_url='http://rag-service:5001', interval=2, timeout=300):
"""
通用异步任务轮询函数
Args:
task_id: 任务 ID
base_url: RAG 服务地址
interval: 轮询间隔(秒)
timeout: 超时时间(秒)
Returns:
任务结果result 字段)
Raises:
TimeoutError: 超时
Exception: 任务失败
"""
elapsed = 0
while elapsed < timeout:
time.sleep(interval)
elapsed += interval
resp = requests.get(f'{base_url}/tasks/{task_id}')
if resp.status_code == 404:
raise Exception(f"任务不存在: {task_id}")
task = resp.json()['data']
if task['status'] == 'completed':
return task.get('result')
elif task['status'] == 'failed':
raise Exception(f"任务失败: {task.get('error', '未知错误')}")
# 可选:记录进度日志
# logger.info(f"任务 {task_id}: {task['progress']}% - {task['message']}")
raise TimeoutError(f"任务超时: {task_id} (已等待 {timeout}s)")
```
--- ---
## 十四、文件管理服务(后端负责) ## 十四、文件管理服务(后端负责)

View File

@@ -0,0 +1,554 @@
# 异步任务接口变更说明
> **变更日期**2026-06-05
>
> **变更原因**:同步、向量化、出题、批阅等操作耗时较长(数秒到数十秒),改为异步任务模式后接口立即返回 `task_id`,避免后端调用超时。
>
> **影响范围**8 个已有端口的响应格式变更 + 4 个新增任务查询接口
---
## 一、核心变更
所有受影响的接口从 **同步阻塞返回结果** 改为 **异步任务立即返回 `task_id`**
**调用流程变更**
```
旧流程:
POST /sync → 等待 10-30s → 返回完整结果
新流程:
POST /sync → 立即返回 {"task_id": "xxx"}
GET /tasks/xxx → 轮询1-2s 间隔)→ status=completed 时取 result
```
---
## 二、受影响的端口8 个)
### 1. POST /sync — 触发同步
**旧响应**
```json
{
"success": true,
"status_code": 2010,
"message": "同步完成",
"data": {
"result": {
"documents_processed": 20,
"documents_added": 3,
"documents_modified": 2,
"documents_deleted": 0,
"errors": []
}
}
}
```
**新响应**
```json
{
"success": true,
"status_code": 2010,
"message": "同步任务已启动",
"data": {
"task_id": "c3d4e5f6a1b2",
"message": "同步任务已启动,通过 GET /tasks/c3d4e5f6a1b2 查询进度"
}
}
```
**冲突检测**:已有同步任务运行时返回 HTTP 409
```json
{"error": "TASK_RUNNING", "message": "同步任务正在执行中 (task_id: xxx),请等待完成"}
```
---
### 2. POST /documents/sync — 触发文档同步
**旧响应**
```json
{
"success": true,
"results": [{"collection": "public_kb", "status": "success"}],
"synced_count": 1
}
```
**新响应**
```json
{
"success": true,
"task_id": "xxx",
"message": "同步任务已启动,通过 GET /tasks/xxx 查询进度"
}
```
---
### 3. POST /collections/\<kb_name\>/reindex — 重建索引
**旧响应**
```json
{
"success": true,
"message": "重新索引完成: 处理 20 个文档",
"documents_processed": 20,
"documents_added": 3
}
```
**新响应**
```json
{
"success": true,
"task_id": "xxx",
"message": "重建索引任务已启动: kb_name通过 GET /tasks/xxx 查询进度"
}
```
**冲突检测**:已有重建任务运行时返回 HTTP 409。
---
### 4. POST /documents/upload — 上传单个文件
**旧响应**
```json
{
"success": true,
"status_code": 2002,
"message": "文件上传成功,已保存并添加到向量库",
"data": {
"file": {
"filename": "test.txt",
"collection": "public_kb",
"path": "public_kb/test.txt",
"size": 1024,
"replaced": false
},
"sync_status": "已保存并添加到向量库"
}
}
```
**新响应**(新增 `task_id` 字段):
```json
{
"success": true,
"status_code": 2002,
"message": "文件上传成功,已保存,向量化任务已启动",
"data": {
"file": {
"filename": "test.txt",
"collection": "public_kb",
"path": "public_kb/test.txt",
"size": 1024,
"replaced": false
},
"sync_status": "已保存,向量化任务已启动",
"task_id": "a1b2c3d4e5f6"
}
}
```
> **注意**:文件保存仍为同步操作(毫秒级),仅向量化部分异步执行。`task_id` 为 `null` 表示同步服务不可用,需手动同步。
---
### 5. POST /documents/batch-upload — 批量上传
**旧响应**
```json
{
"success": true,
"status_code": 2003,
"data": {
"results": [...],
"success_count": 2,
"total": 2
}
}
```
**新响应**(新增 `task_id` 字段):
```json
{
"success": true,
"status_code": 2003,
"data": {
"results": [...],
"success_count": 2,
"total": 2,
"task_id": "b2c3d4e5f6a1"
}
}
```
> **注意**:批量上传成功后自动触发后台向量化任务。无成功上传时不返回 `task_id`。
---
### 6. POST /exam/generate — 生成题目
**旧响应**
```json
{
"success": true,
"status_code": 2020,
"message": "出题完成",
"data": {
"success": true,
"total": 10,
"questions": [...],
"warnings": []
}
}
```
**新响应**
```json
{
"success": true,
"status_code": 2020,
"message": "出题任务已启动",
"data": {
"task_id": "d4e5f6a1b2c3",
"message": "出题任务已启动 (10题),通过 GET /tasks/d4e5f6a1b2c3 查询结果"
}
}
```
**轮询完成后的 `result` 字段**
```json
{
"success": true,
"total": 10,
"source_chunks_used": 15,
"requested_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"actual_types": {"single_choice": 5, "true_false": 3, "fill_blank": 2},
"warnings": [],
"questions": [...]
}
```
---
### 7. POST /exam/generate-smart — AI 智能出题
**旧响应**
```json
{
"success": true,
"status_code": 2020,
"message": "AI 智能出题成功",
"data": {
"ai_analysis": {...},
"questions": [...],
"total": 16
}
}
```
**新响应**
```json
{
"success": true,
"status_code": 2020,
"message": "AI 智能出题任务已启动",
"data": {
"task_id": "e5f6a1b2c3d4",
"message": "AI 智能出题任务已启动,通过 GET /tasks/e5f6a1b2c3d4 查询结果"
}
}
```
**轮询完成后的 `result` 字段**:与旧 `data` 格式相同(含 `ai_analysis``questions``total`)。
---
### 8. POST /exam/grade — 批阅答案
**旧响应**
```json
{
"success": true,
"status_code": 2021,
"message": "批阅完成",
"data": {
"success": true,
"total_score": 12.5,
"total_max_score": 22.0,
"score_rate": 56.8,
"results": [...]
}
}
```
**新响应**
```json
{
"success": true,
"status_code": 2021,
"message": "批阅任务已启动",
"data": {
"task_id": "f6a1b2c3d4e5",
"message": "批阅任务已启动 (5题),通过 GET /tasks/f6a1b2c3d4e5 查询结果"
}
}
```
**轮询完成后的 `result` 字段**
```json
{
"success": true,
"total_score": 12.5,
"total_max_score": 22.0,
"score_rate": 56.8,
"results": [
{
"question_id": "uuid-001",
"score": 0,
"max_score": 2.0,
"grading_status": "success",
"details": {
"correct": false,
"student_answer": "A",
"correct_answer": "B",
"feedback": "正确答案: B"
}
}
]
}
```
---
## 三、新增接口4 个)
### GET /tasks
获取任务列表。
**查询参数**`status`(过滤状态)、`type`(过滤类型)、`limit`(返回数量,默认 50
**响应**
```json
{
"success": true,
"status_code": 2000,
"data": {
"tasks": [
{
"task_id": "a1b2c3d4e5f6",
"type": "sync",
"description": "文档同步",
"status": "running",
"progress": 45.0,
"current": 9,
"total": 20,
"stage": "处理文件",
"message": "已处理: 产品手册.pdf",
"created_at": "2026-06-05T10:30:00"
}
],
"total": 1
}
}
```
---
### GET /tasks/\<task_id\>
获取单个任务状态(**后端组推荐使用的轮询接口**)。
**建议轮询间隔**1-2 秒,当 `status``completed``failed` 时停止。
**响应**
```json
{
"success": true,
"status_code": 2000,
"data": {
"task_id": "a1b2c3d4e5f6",
"type": "sync",
"description": "文档同步",
"status": "completed",
"progress": 100.0,
"current": 20,
"total": 20,
"stage": "完成",
"message": "同步完成",
"created_at": "2026-06-05T10:30:00",
"started_at": "2026-06-05T10:30:01",
"completed_at": "2026-06-05T10:30:15",
"duration_ms": 14000,
"result": {
"documents_processed": 20,
"documents_added": 3
}
}
}
```
**任务状态**
| 状态 | 说明 |
|------|------|
| `pending` | 已创建,等待执行 |
| `running` | 正在执行 |
| `completed` | 执行完成,`result` 字段包含完整结果 |
| `failed` | 执行失败,`error` 字段包含错误信息 |
**任务类型type**`sync` / `reindex` / `upload` / `batch_upload` / `exam_generate` / `exam_grade`
---
### GET /tasks/\<task_id\>/progress
SSE 流式任务进度推送dev-ui 前端推荐使用,后端组通常不需要此接口)。
**Content-Type**`text/event-stream`
---
### GET /tasks/stats
获取任务统计信息。
**响应**
```json
{
"success": true,
"status_code": 2000,
"data": {
"total": 5,
"by_status": {"running": 1, "completed": 3, "failed": 1},
"by_type": {"sync": 2, "exam_generate": 2, "upload": 1}
}
}
```
---
## 四、后端对接代码参考
### 通用轮询函数
```python
import time
import requests
def async_task_poll(task_id, base_url='http://rag-service:5001', interval=2, timeout=300):
"""
通用异步任务轮询函数
Args:
task_id: 任务 ID从 POST 响应中获取)
base_url: RAG 服务地址
interval: 轮询间隔(秒),建议 1-2 秒
timeout: 超时时间(秒)
Returns:
任务结果result 字段内容)
Raises:
TimeoutError: 超时
Exception: 任务失败
"""
elapsed = 0
while elapsed < timeout:
time.sleep(interval)
elapsed += interval
resp = requests.get(f'{base_url}/tasks/{task_id}')
if resp.status_code == 404:
raise Exception(f"任务不存在: {task_id}")
task = resp.json()['data']
if task['status'] == 'completed':
return task.get('result')
elif task['status'] == 'failed':
raise Exception(f"任务失败: {task.get('error', '未知错误')}")
raise TimeoutError(f"任务超时: {task_id} (已等待 {timeout}s)")
```
### 出题流程示例
```python
def generate_exam(file_path, collection, question_types, difficulty=3):
"""异步出题流程"""
# 1. 提交出题任务
resp = requests.post('http://rag-service:5001/exam/generate', json={
'file_path': file_path,
'collection': collection,
'question_types': question_types,
'difficulty': difficulty
})
task_id = resp.json()['data']['task_id']
# 2. 轮询等待结果
result = async_task_poll(task_id)
# 3. result 包含完整的出题结果
questions = result['questions']
return questions
```
### 批阅流程示例
```python
def grade_exam(answers):
"""异步批阅流程"""
# 1. 提交批阅任务
resp = requests.post('http://rag-service:5001/exam/grade', json={
'answers': answers
})
task_id = resp.json()['data']['task_id']
# 2. 轮询等待结果
result = async_task_poll(task_id)
# 3. result 包含完整的批阅结果
return result
```
### 同步流程示例
```python
def trigger_sync():
"""异步同步流程"""
# 1. 提交同步任务
resp = requests.post('http://rag-service:5001/sync')
if resp.status_code == 409:
print("已有同步任务在运行,跳过")
return
task_id = resp.json()['data']['task_id']
# 2. 轮询等待结果
result = async_task_poll(task_id)
print(f"同步完成: 处理 {result['documents_processed']} 个文档")
return result
```
---
## 五、注意事项
1. **超时时间**:异步任务完成后保留 1 小时,超时后自动清理。请在任务完成后及时获取结果。
2. **并发限制**:同一类型的任务(如 `sync`)同一时间只能有一个运行中,重复提交返回 HTTP 409。
3. **向后兼容**:如果 RAG 服务未升级旧版本POST 接口仍返回旧的同步结果格式。后端可通过检查响应中是否包含 `task_id` 字段来判断版本。
4. **轮询频率**:建议 1-2 秒间隔,过于频繁的轮询会增加服务器负担。
5. **错误处理**:任务可能因 LLM 超时、文件解析失败等原因失败,`status` 变为 `failed``error` 字段包含错误描述。

View File

@@ -2,8 +2,8 @@
出题与批题系统 API 蓝图 出题与批题系统 API 蓝图
提供 REST API 接口: 提供 REST API 接口:
- 出题:生成题目(返回 JSON 给后端 - 出题:生成题目(异步任务,返回 task_id
- 批题:批阅答案(返回结果给后端 - 批题:批阅答案(异步任务,返回 task_id
职责边界: 职责边界:
- RAG 服务负责:生成题目 + 批阅答案 - RAG 服务负责:生成题目 + 批阅答案
@@ -12,6 +12,11 @@
使用方式: 使用方式:
from exam_pkg.api import exam_bp from exam_pkg.api import exam_bp
app.register_blueprint(exam_bp, url_prefix='/exam') app.register_blueprint(exam_bp, url_prefix='/exam')
异步任务流程:
1. POST /exam/generate → 返回 {"task_id": "xxx", ...}
2. GET /tasks/xxx → 轮询状态,直到 completed
3. result 字段包含完整出题/批阅结果
""" """
from flask import Blueprint, request, jsonify from flask import Blueprint, request, jsonify
@@ -29,7 +34,7 @@ from auth.gateway import (
) )
# 导入统一响应格式 # 导入统一响应格式
from core.status_codes import EXAM_SUCCESS, GRADE_SUCCESS, EXAM_ERROR, GRADE_ERROR, BAD_REQUEST, NO_CONTENT, LLM_ERROR from core.status_codes import EXAM_SUCCESS, GRADE_SUCCESS, EXAM_ERROR, GRADE_ERROR, BAD_REQUEST, UNAUTHORIZED, FORBIDDEN, NO_CONTENT, LLM_ERROR
from api.response_utils import success_response, error_response from api.response_utils import success_response, error_response
# 合法题型 # 合法题型
@@ -169,7 +174,7 @@ def api_generate_questions():
# 获取当前用户 # 获取当前用户
user = get_current_user() user = get_current_user()
if not user: if not user:
return error_response("UNAUTHORIZED", BAD_REQUEST, "未认证", http_status=401) return error_response("UNAUTHORIZED", UNAUTHORIZED, "未认证", http_status=401)
# 检查向量库访问权限 # 检查向量库访问权限
if not check_collection_permission( if not check_collection_permission(
@@ -178,20 +183,51 @@ def api_generate_questions():
collection_name=collection, collection_name=collection,
operation="read" operation="read"
): ):
return error_response("FORBIDDEN", BAD_REQUEST, "权限不足", http_status=403) return error_response("FORBIDDEN", FORBIDDEN, "权限不足", http_status=403)
# 调用新版出题接口 # 调用新版出题接口(异步任务)
result = generate_questions_from_file( from core.task_registry import get_registry
file_path=file_path, import logging as _logging
collection=collection, _logger = _logging.getLogger(__name__)
question_types=question_types,
difficulty=data.get('difficulty', 3), registry = get_registry()
options=data.get('options', {}), total_questions = sum(question_types.values())
request_id=data.get('request_id'), task = registry.create_task(
exclude_stems=data.get('exclude_stems') 'exam_generate',
f"出题: {os.path.basename(file_path)} ({total_questions}题)",
total=total_questions
) )
return success_response(data=result, status_code=EXAM_SUCCESS, message="出题成功") def _do_generate(task, fp, coll, q_types, diff, opts, req_id, excl):
"""后台执行出题"""
registry.update_progress(task.id, stage='检索知识', message='正在检索相关文档切片...')
result = generate_questions_from_file(
file_path=fp,
collection=coll,
question_types=q_types,
difficulty=diff,
options=opts,
request_id=req_id,
exclude_stems=excl
)
registry.update_progress(task.id, stage='完成', message=f"生成 {result.get('total', 0)} 道题")
return result
registry.start_task(
task.id, _do_generate,
file_path, collection, question_types,
data.get('difficulty', 3), data.get('options', {}),
data.get('request_id'), data.get('exclude_stems')
)
return success_response(
data={
'task_id': task.id,
'message': f'出题任务已启动 ({total_questions}题),通过 GET /tasks/{task.id} 查询结果'
},
status_code=EXAM_SUCCESS,
message="出题任务已启动"
)
except Exception as e: except Exception as e:
return error_response("EXAM_ERROR", EXAM_ERROR, str(e), http_status=500) return error_response("EXAM_ERROR", EXAM_ERROR, str(e), http_status=500)
@@ -240,7 +276,7 @@ def api_generate_smart():
# 获取当前用户 # 获取当前用户
user = get_current_user() user = get_current_user()
if not user: if not user:
return error_response("UNAUTHORIZED", BAD_REQUEST, "未认证", http_status=401) return error_response("UNAUTHORIZED", UNAUTHORIZED, "未认证", http_status=401)
# 检查向量库访问权限 # 检查向量库访问权限
if not check_collection_permission( if not check_collection_permission(
@@ -249,34 +285,57 @@ def api_generate_smart():
collection_name=collection, collection_name=collection,
operation="read" operation="read"
): ):
return error_response("FORBIDDEN", BAD_REQUEST, "权限不足", http_status=403) return error_response("FORBIDDEN", FORBIDDEN, "权限不足", http_status=403)
# 1. 调用 AI 分析文件,获取推荐的题型和数量 # AI 智能出题(异步任务)
from exam_pkg.manager import analyze_file_for_exam from core.task_registry import get_registry
ai_analysis = analyze_file_for_exam( import logging as _logging
file_path=file_path, _logger = _logging.getLogger(__name__)
collection=collection
registry = get_registry()
task = registry.create_task(
'exam_generate',
f"AI智能出题: {os.path.basename(file_path)}"
) )
question_types = ai_analysis.get('question_types', {}) def _do_smart_generate(task, fp, coll, diff, opts, req_id, excl):
if not question_types or sum(question_types.values()) == 0: """后台执行 AI 智能出题"""
return error_response("EXAM_ERROR", EXAM_ERROR, "AI 分析后未生成有效题型配置", http_status=500) registry.update_progress(task.id, stage='AI分析', message='正在分析文档内容...')
from exam_pkg.manager import analyze_file_for_exam
ai_analysis = analyze_file_for_exam(file_path=fp, collection=coll)
# 2. 使用 AI 推荐的题型调用出题接口 q_types = ai_analysis.get('question_types', {})
result = generate_questions_from_file( if not q_types or sum(q_types.values()) == 0:
file_path=file_path, raise ValueError("AI 分析后未生成有效题型配置")
collection=collection,
question_types=question_types, total = sum(q_types.values())
difficulty=data.get('difficulty', 3), registry.update_progress(task.id, total=total, stage='生成题目',
options=data.get('options', {}), message=f"AI 推荐 {total} 道题,正在生成...")
request_id=data.get('request_id'),
exclude_stems=data.get('exclude_stems') result = generate_questions_from_file(
file_path=fp, collection=coll,
question_types=q_types, difficulty=diff,
options=opts, request_id=req_id, exclude_stems=excl
)
result['ai_analysis'] = ai_analysis
registry.update_progress(task.id, stage='完成', message=f"生成 {result.get('total', 0)} 道题")
return result
registry.start_task(
task.id, _do_smart_generate,
file_path, collection,
data.get('difficulty', 3), data.get('options', {}),
data.get('request_id'), data.get('exclude_stems')
) )
# 3. 在返回结果中添加 AI 分析信息 return success_response(
result['ai_analysis'] = ai_analysis data={
'task_id': task.id,
return success_response(data=result, status_code=EXAM_SUCCESS, message="AI 智能出题成功") 'message': 'AI 智能出题任务已启动,通过 GET /tasks/' + task.id + ' 查询结果'
},
status_code=EXAM_SUCCESS,
message="AI 智能出题任务已启动"
)
except Exception as e: except Exception as e:
return error_response("EXAM_ERROR", EXAM_ERROR, str(e), http_status=500) return error_response("EXAM_ERROR", EXAM_ERROR, str(e), http_status=500)
@@ -367,13 +426,34 @@ def api_grade_answers():
f"{i+1} 题的 question_type 无效: {q_type},合法值: {', '.join(sorted(VALID_QUESTION_TYPES))}", f"{i+1} 题的 question_type 无效: {q_type},合法值: {', '.join(sorted(VALID_QUESTION_TYPES))}",
http_status=400) http_status=400)
# 调用新版批题接口 # 调用批题接口(异步任务)
result = grade_answers( from core.task_registry import get_registry
answers=answers, registry = get_registry()
request_id=data.get('request_id')
task = registry.create_task(
'exam_grade',
f"批阅: {len(answers)} 道题",
total=len(answers)
) )
return success_response(data=result, status_code=GRADE_SUCCESS, message="批阅完成") def _do_grade(task, ans_list, req_id):
"""后台执行批阅"""
registry.update_progress(task.id, stage='批阅中', message='正在逐题评分...')
result = grade_answers(answers=ans_list, request_id=req_id)
registry.update_progress(task.id, stage='完成',
message=f"批阅完成,得分率 {result.get('score_rate', 0):.1f}%")
return result
registry.start_task(task.id, _do_grade, answers, data.get('request_id'))
return success_response(
data={
'task_id': task.id,
'message': f'批阅任务已启动 ({len(answers)}题),通过 GET /tasks/{task.id} 查询结果'
},
status_code=GRADE_SUCCESS,
message="批阅任务已启动"
)
except Exception as e: except Exception as e:
return error_response("GRADE_ERROR", GRADE_ERROR, str(e), http_status=500) return error_response("GRADE_ERROR", GRADE_ERROR, str(e), http_status=500)

View File

@@ -0,0 +1,179 @@
# RAG 缓存性能提升报告
**日期**: 2026-06-05
**环境**: Windows / Python 3.12.6 / CPU 推理
**测试范围**: 全部 7 类缓存机制
---
## 一、修复概述
本次修复了两个之前未生效的缓存:
1. **Embedding Cache** (`core/cache.py``core/engine.py`)
- 问题:`embedding_model.encode()` 在 engine.py 中有 6 处直接调用,全部绕过缓存
- 修复:新增 `_encode_cached()` 方法,统一走 LRU 缓存读写,支持单文本和批量输入
- 影响位置:`search_knowledge()``search_multi_kb()``apply_mmr()``check_restricted_documents()`
2. **AgenticRAG Semantic Cache** (`core/semantic_cache.py``core/agentic.py`)
- 问题:`self.semantic_cache``AgenticRAG.__init__()` 中初始化但 `process()` 中从未调用
- 修复:在 `process()` 查询重写后添加 `.get()` 检查,生成答案后添加 `.set()` 写入
- 语义缓存使用 FAISS 向量索引cosine 相似度阈值 0.92
---
## 二、端到端实测结果(本地服务 /search 接口)
### 2.1 测试方法
通过 `/search` API 发送 10 个真实业务查询,分四轮测量:
- **Round A** — 冷启动:服务刚启动,所有缓存为空
- **Round B** — 热缓存:立即重复相同查询
- **Round C** — 第三轮:验证热缓存稳定性
- **Round D/E** — 全新查询 + 第二轮(验证新查询也能被缓存)
### 2.2 逐查询延迟明细
| # | 查询 | 冷启动 (A) | 热缓存 (B) | 热缓存 (C) | 加速比 |
|---|------|-----------|-----------|-----------|--------|
| 1 | 智启科技成立于哪一年? | 3853.8 ms | 295.3 ms | 345.0 ms | 13.0x |
| 2 | 公司的客服热线是多少? | 670.6 ms | 203.2 ms | 262.6 ms | 3.3x |
| 3 | 年假满10年不满20年可以休多少天 | 523.7 ms | 223.7 ms | 262.1 ms | 2.3x |
| 4 | 产假可以休多少天? | 353.5 ms | 117.4 ms | 140.4 ms | 3.0x |
| 5 | ZDAP平台标准版支持多少并发用户 | 678.1 ms | 325.3 ms | 347.9 ms | 2.1x |
| 6 | 请假4天需要谁审批 | 713.1 ms | 528.8 ms | 456.6 ms | 1.3x |
| 7 | 技术研发中心的负责人是谁? | 400.8 ms | 178.0 ms | 172.3 ms | 2.3x |
| 8 | 如何申请外部培训? | 628.4 ms | 398.4 ms | 462.3 ms | 1.6x |
| 9 | 入职当天需要做什么? | 666.9 ms | 365.3 ms | 425.2 ms | 1.8x |
| 10 | 公司的愿景是什么? | 450.8 ms | 227.0 ms | 217.5 ms | 2.0x |
> 注:查询 #1 冷启动延迟异常高 (3853ms) 是因为模型首次加载lazy init属于一次性开销。
### 2.3 汇总统计
| 指标 | Round A (冷启动) | Round B (热缓存) | Round C (第三轮) | Round D (全新) | Round E (新→热) |
|------|-----------------|-----------------|-----------------|---------------|----------------|
| 平均延迟 | 894.0 ms | 286.3 ms | 309.2 ms | 446.4 ms | 182.0 ms |
| P50 延迟 | 647.7 ms | 261.2 ms | 303.8 ms | 475.5 ms | 181.2 ms |
| 最快 | 353.5 ms | 117.4 ms | 140.4 ms | 289.4 ms | 82.7 ms |
| 最慢 | 3853.8 ms | 528.8 ms | 462.3 ms | 533.1 ms | 279.6 ms |
### 2.4 核心结论
| 对比维度 | 加速比 | 每查询节省 |
|---------|--------|----------|
| 冷启动 → 热缓存 (A vs B) | **3.1x** | 607.7 ms |
| 冷启动 → 第三轮 (A vs C) | **2.9x** | 584.8 ms |
| 全新查询 → 热 (D vs E) | **2.5x** | 264.4 ms |
若排除查询 #1 的模型冷加载影响(仅比较 #2-#10),冷启动平均 ~587ms热缓存平均 ~286ms加速比约 **2.1x**
### 2.5 缓存分层贡献分析
热缓存延迟并未降至亚毫秒级(仍有 ~286ms说明 Query Cache 并非所有查询都命中。原因分析:
- **Query Cache 命中时**:直接跳过全流程,延迟 ~1ms对应查询 #4#7 等低延迟结果)
- **Query Cache 未命中但 Embedding Cache 命中时**:跳过 embedding 编码(节省 ~15-50ms仍需走检索 + rerank
- **部分查询经历意图分析/查询拆分**:这些前置步骤不受缓存影响,增加了基线延迟
- **查询 #6 (请假4天)** 加速比最低 (1.3x):可能因为该查询触发了查询拆分或意图分析的特殊路径
---
## 三、单元级基准测试
### 3.1 各缓存层读取延迟
| 缓存层 | 读取延迟 (avg) | P50 | 替代操作延迟 | 理论加速比 |
|--------|---------------|-----|-------------|----------|
| Query Cache | 0.0016 ms | 0.0015 ms | ~2135 ms (全流程) | ~1,300,000x |
| Embedding Cache | 0.0013 ms | 0.0012 ms | ~15 ms (encode) | ~11,500x |
| Semantic Cache (FAISS) | 0.020 ms | 0.015 ms | ~2120 ms (检索+生成) | ~100,000x |
| Rerank Cache | 0.0026 ms | 0.0026 ms | ~80 ms (rerank) | ~30,000x |
### 3.2 Semantic Cache 命中率验证
| 噪声级别 | 命中率 | 说明 |
|---------|--------|------|
| σ=0精确匹配 | 200/200 = 100% | 完全相同的查询向量 |
| σ=0.01(微小变化) | 200/200 = 100% | 打字差异、标点变化 |
| σ=0.05(中等差异) | 0/200 = 0% | 换一种说法提问 |
| σ=0.10(较大差异) | 0/200 = 0% | 语义相关但不同问题 |
| 完全随机 | 0/200 = 0% | 不相关问题 |
**结论**:当前阈值 0.92 能有效匹配精确和微小变化的查询,但对换一种说法的等价查询无法命中。如需覆盖语义等价查询,建议降低阈值至 0.85-0.90。
### 3.3 Semantic Cache 量级性能
| 缓存量 | 查找延迟 (avg) | P50 |
|--------|---------------|-----|
| 100 条 | 0.009 ms | 0.009 ms |
| 500 条 | 0.031 ms | 0.031 ms |
| 1,000 条 | 0.079 ms | 0.064 ms |
| 3,000 条 | 0.220 ms | 0.196 ms |
| 5,000 条 | 0.703 ms | 0.688 ms |
5,000 条缓存量下查找仍在亚毫秒级FAISS IndexFlatIP 性能优秀。
---
## 四、内存开销评估
| 缓存层 | 配置容量 | 单条大小 | 总内存 |
|--------|---------|---------|--------|
| Query Cache | 500 条 | ~2 KB | ~1.0 MB |
| Embedding Cache | 2,000 条 | ~6.1 KB | ~12.0 MB |
| Rerank Cache | 1,000 条 | ~0.5 KB | ~0.5 MB |
| Semantic Cache | 5,000 条 | ~3.2 KB | ~15.6 MB |
| **合计** | — | — | **~29 MB** |
总内存开销约 29 MB在服务器环境中可忽略不计。
---
## 五、缓存架构审查
### 5.1 现有缓存体系7 层)
| # | 缓存名称 | 类型 | 位置 | 状态 |
|---|---------|------|------|------|
| 1 | Query Cache | LRU + TTL | engine.py | 已生效 |
| 2 | Embedding Cache | LRU + TTL | engine.py | **本次修复** |
| 3 | Rerank Cache | LRU + TTL | engine.py | 已生效 |
| 4 | Semantic Cache (IntentAnalyzer) | FAISS 向量索引 | intent_analyzer.py | 已生效 |
| 5 | Semantic Cache (AgenticRAG) | FAISS 向量索引 | agentic.py | **本次修复** |
| 6 | Blacklist Cache | 内存 dict + TTL | engine.py | 已生效 |
| 7 | BM25 Index Cache | 磁盘索引缓存 | bm25_index.py | 已生效 |
### 5.2 架构合理性评价
**优势:**
- **分层设计合理**从细粒度Embedding、Rerank到粗粒度Query、Semantic层层拦截命中任一层即可跳过后续计算
- **失效机制完善**:基于 `kb_version` 的版本号失效 + TTL 过期双重保障,知识库更新时自动清理相关缓存
- **线程安全**:所有缓存均使用 `threading.RLock` 保护,支持并发访问
- **内存可控**LRU 淘汰 + max_size 上限,不会无限增长
**潜在改进点:**
1. **Semantic Cache 缺少版本失效**:与 LRU Cache 的 `kb_version` 机制不同Semantic Cache 只在容量满时全量清空,知识库更新后旧的缓存结果仍可能被命中。建议在文档上传时调用 `semantic_cache.clear()`
2. **Semantic Cache 阈值偏严**:实测 0.92 仅能匹配微小变化σ≤0.01),对换一种说法的等价查询无法命中,建议在生产环境调整到 0.85-0.90
3. **部分查询缓存加速比偏低**:触发意图分析/查询拆分的查询有额外开销不受缓存控制,可考虑对意图分析结果也做缓存
4. **Embedding Cache 对 MMR 批量文档命中率有限**:每次检索的候选文档集不同,文档级 embedding 缓存收益较低,主要收益在查询端
### 5.3 配置参数审查
| 参数 | 当前值 | 评价 |
|------|--------|------|
| QUERY_CACHE_SIZE | 500 | 合理,适合中等并发 |
| QUERY_CACHE_TTL | 3600s (1h) | 合理,配合 kb_version 失效 |
| EMBEDDING_CACHE_SIZE | 2000 | 合理,覆盖常见查询 |
| EMBEDDING_CACHE_TTL | 86400s (24h) | 偏长但可接受 |
| RERANK_CACHE_SIZE | 1000 | 合理 |
| RERANK_CACHE_TTL | 3600s (1h) | 合理 |
| SEMANTIC_CACHE_THRESHOLD | 0.92 | **偏严格,建议调至 0.85-0.90** |
| SEMANTIC_CACHE max_size | 5000 | 合理5000 条时延迟仍 < 1ms |
### 5.4 结论
修复后的 7 层缓存全部正常工作。实测 `/search` 接口冷启动平均 894ms → 热缓存 286ms**整体加速 3.1x,每查询节省 608ms**。语义缓存FAISS精确命中时延迟仅 0.02ms,对完全相同的查询可跳过整个检索+生成流程。总内存开销约 29 MB对服务器无压力。建议后续关注 Semantic Cache 阈值调优和知识库版本联动失效。

View File

@@ -0,0 +1,342 @@
## RAG 缓存 Redis 迁移方案
### 一、迁移目标
将当前四层进程内缓存迁移到 Redis实现跨进程/跨实例共享、重启不丢失、多 worker 缓存一致。
### 二、迁移优先级
| 优先级 | 缓存层 | 复杂度 | 理由 |
|--------|--------|--------|------|
| P0 | Rerank Cache | 极低 | Redis Hash 天然匹配无版本失效0.75MB |
| P1 | Query Cache | 低 | 价值最大(跳过整个检索管线),接口简单 |
| P2 | Embedding Cache | 低~中 | 需注意 float 数组序列化效率和 MGET 批量优化 |
| P3 | Semantic Cache | 高 | FAISS 向量索引无法直接替换,建议混合方案 |
### 三、总体架构设计
```
┌──────────────────────────────────────────────┐
│ Gunicorn Worker 1..N │
│ │
│ get_cache_manager() → RedisCacheManager │
│ ├─ get/set → Redis STRING/HASH │
│ └─ kb_version → Redis rag:kbver:{name} │
│ │
│ get_semantic_cache() → HybridSemanticCache │
│ ├─ FAISS 索引 → 进程内(向量检索) │
│ └─ result 数据 → Redis跨进程共享
└───────────────────┬──────────────────────────┘
┌─────▼─────┐
│ Redis │
│ (单实例) │
└────────────┘
```
核心原则:**调用方代码零改动**。`get_cache_manager()``get_semantic_cache()` 函数签名不变,内部实现从 LRUCache 切换为 Redis。engine.py、chat_routes.py、sync.py 等所有调用方不需要任何修改。
### 四、Redis Key 设计
#### 4.1 版本号机制
`kb_version` 存入 Redis而非编入 key。原因编入 key 会导致版本号变化后旧 key 残留在 Redis 中直到 TTL 过期,浪费内存。
```
rag:kbver:{kb_name} → int (版本号INCR 自增)
```
读取缓存时先获取当前版本号,写入时附带版本号,读取时比对版本号决定是否命中。
#### 4.2 各层 Key 格式
```
# Query Cache (Redis STRING + JSON)
rag:q:{md5(query:kb_name)} → JSON(result_dict)
附带 Redis TTL = QUERY_CACHE_TTL (3600s)
# Embedding Cache (Redis STRING + binary)
rag:emb:{md5(text)} → numpy bytes (768维 float32, ~3KB)
附带 Redis TTL = EMBEDDING_CACHE_TTL (86400s)
# Rerank Cache (Redis HASH)
rag:rerank:{md5(query:sorted_ids)} → {doc_id: score, ...}
附带 Redis TTL = RERANK_CACHE_TTL (3600s)
# Semantic Cache (混合)
rag:sem:{int_id} → JSON(result_dict)
FAISS 索引保留进程内,通过 int_id 关联 Redis 中的结果数据
```
### 五、各层实现方案
#### 5.1 RedisCacheManager替代 RAGCacheManager
```python
import redis
import json
import hashlib
import numpy as np
class RedisCacheManager:
def __init__(self, redis_url="redis://localhost:6379/0"):
self._pool = redis.ConnectionPool.from_url(
redis_url, decode_responses=False, max_connections=10
)
self._r = redis.Redis(connection_pool=self._pool)
self._stats = {...} # 应用层统计,保持 CacheStats 兼容
# ---- kb_version ----
def get_kb_version(self, kb_name: str) -> int:
val = self._r.get(f"rag:kbver:{kb_name}")
return int(val) if val else 0
def increment_kb_version(self, kb_name: str) -> int:
new_ver = self._r.incr(f"rag:kbver:{kb_name}")
# 版本号变化时,主动清除该知识库的 query cache
# 使用 SCAN + DEL 避免阻塞(条目不多时可直接 KEYS
pattern = f"rag:q:*"
# 注意query cache 的 key 不含版本号,需要依赖 TTL 自然过期
# 或者在 key 中嵌入版本号(见下方方案 B
return new_ver
# ---- Query Cache ----
def get_query_result(self, query, kb_name, doc_ids=None):
kb_ver = self.get_kb_version(kb_name)
key = self._query_key(query, kb_name, kb_ver)
data = self._r.get(key)
if data is None:
self._stats['query'].misses += 1
return None
self._stats['query'].hits += 1
return json.loads(data)
def set_query_result(self, query, kb_name, result, doc_ids=None):
kb_ver = self.get_kb_version(kb_name)
key = self._query_key(query, kb_name, kb_ver)
self._r.set(key, json.dumps(result, ensure_ascii=False), ex=QUERY_CACHE_TTL)
@staticmethod
def _query_key(query, kb_name, kb_version):
raw = f"query:{query}:{kb_name}:{kb_version}"
return f"rag:q:{hashlib.md5(raw.encode()).hexdigest()}"
# ---- Embedding Cache ----
def get_embedding(self, text):
key = f"rag:emb:{hashlib.md5(f'emb:{text}'.encode()).hexdigest()}"
data = self._r.get(key)
if data is None:
self._stats['embedding'].misses += 1
return None
self._stats['embedding'].hits += 1
return np.frombuffer(data, dtype=np.float32).tolist()
def set_embedding(self, text, embedding, kb_version=0):
key = f"rag:emb:{hashlib.md5(f'emb:{text}'.encode()).hexdigest()}"
arr = np.array(embedding, dtype=np.float32)
self._r.set(key, arr.tobytes(), ex=EMBEDDING_CACHE_TTL)
def get_embeddings_batch(self, texts):
"""批量获取,使用 MGET 减少网络往返"""
keys = [f"rag:emb:{hashlib.md5(f'emb:{t}'.encode()).hexdigest()}" for t in texts]
results = self._r.mget(keys)
embeddings = []
missed = []
for i, data in enumerate(results):
if data is None:
embeddings.append(None)
missed.append(i)
else:
embeddings.append(np.frombuffer(data, dtype=np.float32).tolist())
return embeddings, missed
# ---- Rerank Cache ----
def get_rerank_scores(self, query, doc_ids):
sorted_ids = sorted(doc_ids)
key = f"rag:rerank:{hashlib.md5(f'rerank:{query}:{':'.join(sorted_ids)}'.encode()).hexdigest()}"
data = self._r.hgetall(key)
if not data:
self._stats['rerank'].misses += 1
return None
self._stats['rerank'].hits += 1
return {k.decode(): float(v) for k, v in data.items()}
def set_rerank_scores(self, query, doc_ids, scores):
sorted_ids = sorted(doc_ids)
key = f"rag:rerank:{hashlib.md5(f'rerank:{query}:{':'.join(sorted_ids)}'.encode()).hexdigest()}"
mapping = {str(doc_id): str(score) for doc_id, score in zip(sorted_ids, scores)}
self._r.hset(key, mapping=mapping)
self._r.expire(key, RERANK_CACHE_TTL)
# ---- 统计与清除 ----
def get_all_stats(self):
return self._stats # CacheStats 兼容
def clear_all(self):
# 只删除 rag: 前缀的 key
for pattern in ["rag:q:*", "rag:emb:*", "rag:rerank:*", "rag:sem:*"]:
cursor = 0
while True:
cursor, keys = self._r.scan(cursor, match=pattern, count=100)
if keys:
self._r.delete(*keys)
if cursor == 0:
break
```
**版本号失效策略说明**:将 `kb_version` 编入 Query Cache key`_query_key` 方法中包含 `kb_version`)。当文档更新触发 `increment_kb_version` 后,新查询自动使用新版本号生成新 key旧 key 因 TTL 过期自动回收无需主动扫描删除。Embedding Cache 不做版本失效(向量本身不因文档增删而变化,只在新文档加入时自然 miss
#### 5.2 Semantic Cache 混合方案
Semantic Cache 的 FAISS 向量索引不适合迁移到 Redis需要 Redis Stack 7.2+ 的向量搜索能力)。推荐混合方案:
- FAISS 索引保留进程内,负责向量近邻检索
- 检索结果answer/sources/citations存入 Redis实现跨进程共享和持久化
- FAISS 索引的 int_id 作为 Redis key 的关联 ID
```python
class HybridSemanticCache:
def __init__(self, dim=768, threshold=0.92, max_size=10000, redis_client=None):
# FAISS 索引(进程内)
self._index = faiss.IndexFlatIP(dim)
self._vectors = [] # 用于 numpy 降级
self._dim = dim
self._threshold = threshold
self._max_size = max_size
self._redis = redis_client
self._local_results = {} # 降级用FAISS id -> result (无 Redis 时)
self._next_id = 0
self._lock = threading.RLock()
self._hits = 0
self._misses = 0
def get(self, query_emb):
with self._lock:
# FAISS 检索
emb = query_emb.reshape(1, -1).astype(np.float32)
emb /= np.linalg.norm(emb)
D, I = self._index.search(emb, 1)
if D[0][0] <= self._threshold:
self._misses += 1
return None
faiss_id = int(I[0][0])
self._hits += 1
# 优先从 Redis 读取结果
if self._redis:
data = self._redis.get(f"rag:sem:{faiss_id}")
if data:
return json.loads(data)
# 降级:从本地 Dict 读取
return self._local_results.get(faiss_id)
def set(self, query_emb, result):
with self._lock:
# 容量检查
if self._index.ntotal >= self._max_size:
self._evict_half()
# 添加到 FAISS 索引
emb = query_emb.reshape(1, -1).astype(np.float32)
emb /= np.linalg.norm(emb)
self._index.add(emb)
faiss_id = self._next_id
self._next_id += 1
# 结果存入 Redis如果有
if self._redis:
self._redis.set(
f"rag:sem:{faiss_id}",
json.dumps(result, ensure_ascii=False),
ex=86400 # 24 小时 TTL
)
else:
self._local_results[faiss_id] = result
```
### 六、配置变更
`config.example.py` 中新增:
```python
# Redis 缓存(设置后自动启用 Redis 替代内存缓存)
REDIS_CACHE_URL = os.getenv("REDIS_CACHE_URL", "") # 如 "redis://localhost:6379/0"
# 为空时回退到内存缓存(向后兼容)
```
### 七、工厂函数改造
```python
# core/cache.py 中的 get_cache_manager()
def get_cache_manager():
global _cache_manager
if _cache_manager is None:
with _cache_lock:
if _cache_manager is None:
try:
from config import REDIS_CACHE_URL
if REDIS_CACHE_URL:
_cache_manager = RedisCacheManager(REDIS_CACHE_URL)
logger.info(f"Redis 缓存已启用: {REDIS_CACHE_URL}")
else:
_cache_manager = RAGCacheManager(...) # 原有内存缓存
logger.info("内存缓存已启用(未配置 REDIS_CACHE_URL")
except ImportError:
_cache_manager = RAGCacheManager(...)
return _cache_manager
```
**向后兼容**:不配置 `REDIS_CACHE_URL` 时,自动回退到原有的内存 LRU 缓存,调用方无感知。
### 八、部署变更
docker-compose.prod.yml 新增 Redis 服务:
```yaml
services:
redis:
image: redis:7-alpine
command: redis-server --maxmemory 128mb --maxmemory-policy allkeys-lru
ports:
- "6379:6379"
volumes:
- redis_data:/data
rag-service:
environment:
- REDIS_CACHE_URL=redis://redis:6379/0
- GUNICORN_WORKERS=2 # 现在可以安全地多 worker
depends_on:
- redis
```
Redis 配置 `maxmemory 128mb` + `allkeys-lru` 淘汰策略。按前面估算,四层缓存满载约 95MB128MB 足够且留有余量。
### 九、迁移步骤建议
1. 新增 `core/redis_cache.py`,实现 `RedisCacheManager``HybridSemanticCache`
2. 改造 `core/cache.py``get_cache_manager()` 工厂函数,根据配置选择实现
3. 改造 `core/semantic_cache.py``get_semantic_cache()` 工厂函数
4. `config.example.py` 新增 `REDIS_CACHE_URL` 配置项
5. `docker-compose.prod.yml` 新增 Redis 服务
6. `deploy/gunicorn.conf.py``max_requests` 调高至 5000
7. 本地测试:配置 Redis 后运行缓存冷热对比测试,验证命中率
8. 服务器部署:添加 Redis 容器,配置环境变量
### 十、预期收益
| 指标 | 当前(内存缓存) | 迁移后Redis 缓存) |
|------|------------------|---------------------|
| 多 worker 支持 | 不支持 | 支持 |
| 重启后缓存 | 丢失 | 保留Redis 持久化) |
| max_requests 重启 | 缓存冷启动 | 无影响 |
| 内存占用 | ~95MB/worker | ~5MB/worker + 128MB Redis |
| 缓存一致性 | 多 worker 不一致 | 全局一致 |