init: RAG 知识库服务初始提交
- 后端 API（Flask + Gunicorn） - RAG 引擎（混合检索 + 云端 Reranker + 引用溯源） - 文档解析（MinerU + 多格式支持） - Docker 生产部署配置 - 排除前端项目、敏感配置、模型文件
This commit is contained in:
lacerate551
987
api/document_routes.py
Normal file
987
api/document_routes.py
Normal file
@@ -0,0 +1,987 @@
|
||||
"""
|
||||
文档管理 API
|
||||
|
||||
本模块提供文档上传、查询、删除等管理功能，包括：
|
||||
- 单文件和批量上传
|
||||
- 文档列表查询
|
||||
- 文档切片管理
|
||||
|
||||
路由列表:
|
||||
POST /documents/upload : 上传文件（单个）
|
||||
POST /documents/batch-upload : 批量上传文件
|
||||
GET /documents/list : 文档列表
|
||||
GET /documents/<doc_id>/status : 文件处理状态
|
||||
PUT /documents/<doc_id> : 更新文件
|
||||
DELETE /documents/<doc_path> : 删除文档
|
||||
GET /documents/<doc_id>/chunks : 查看文件切片
|
||||
GET /documents/<doc_id>/preview : 文档预览（支持按切片序号跳转）
|
||||
|
||||
切片管理:
|
||||
POST /chunks : 新增切片
|
||||
PUT /chunks/<chunk_id> : 修改切片
|
||||
DELETE /chunks/<chunk_id> : 删除切片
|
||||
|
||||
架构说明:
|
||||
- 权限验证由后端网关完成，RAG 服务不做权限判断
|
||||
- 文档存储在 documents/<collection>/ 目录下
|
||||
- 上传后自动触发向量化（如果同步服务可用）
|
||||
|
||||
Example:
|
||||
# 上传文件
|
||||
curl -X POST http://localhost:5001/documents/upload \\
|
||||
-H "Authorization: Bearer mock-token-admin" \\
|
||||
-F "file=@report.pdf" \\
|
||||
-F "collection=public_kb"
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple, Any, List, Dict
|
||||
from flask import Blueprint, request, jsonify
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from werkzeug.utils import secure_filename
|
||||
from auth.gateway import require_gateway_auth
|
||||
from core.status_codes import (
|
||||
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST,
|
||||
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION,
|
||||
FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR
|
||||
)
|
||||
from api.response_utils import success_response, error_response
|
||||
|
||||
document_bp = Blueprint('document', __name__)
|
||||
|
||||
# 文件限制
|
||||
ALLOWED_EXTENSIONS = {'.pdf', '.docx', '.doc', '.xlsx', '.txt'}
|
||||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
|
||||
|
||||
def safe_filename(filename: str) -> str:
|
||||
"""
|
||||
安全文件名处理 - 保留中文，仅移除危险字符
|
||||
|
||||
与 werkzeug.secure_filename() 不同，此函数保留 Unicode 字符（包括中文），
|
||||
仅移除路径分隔符和其他可能导致路径遍历攻击的字符。
|
||||
|
||||
Args:
|
||||
filename: 原始文件名
|
||||
|
||||
Returns:
|
||||
安全的文件名；无效输入返回空字符串
|
||||
|
||||
Example:
|
||||
>>> safe_filename("财务报告.pdf")
|
||||
'财务报告.pdf'
|
||||
>>> safe_filename("../../../etc/passwd")
|
||||
'etc_passwd'
|
||||
"""
|
||||
if not filename:
|
||||
return ""
|
||||
|
||||
# 移除路径分隔符和危险字符
|
||||
dangerous_chars = ['/', '\\', '..', '\x00', '\n', '\r', '\t']
|
||||
safe_name = filename
|
||||
for char in dangerous_chars:
|
||||
safe_name = safe_name.replace(char, '_')
|
||||
|
||||
# 移除首尾空格和点
|
||||
safe_name = safe_name.strip(' .')
|
||||
|
||||
# 如果文件名为空或只有扩展名，返回空
|
||||
if not safe_name or safe_name.startswith('.') and safe_name.count('.') == 1:
|
||||
return ""
|
||||
|
||||
return safe_name
|
||||
|
||||
# 延迟初始化缓存
|
||||
_kb_manager = None
|
||||
_kb_checked = False
|
||||
|
||||
|
||||
def _get_kb_manager() -> Optional[Any]:
|
||||
"""
|
||||
获取知识库管理器（延迟加载）
|
||||
|
||||
Returns:
|
||||
知识库管理器实例，导入失败返回 None
|
||||
"""
|
||||
global _kb_manager, _kb_checked
|
||||
if not _kb_checked:
|
||||
try:
|
||||
from knowledge.manager import get_kb_manager
|
||||
_kb_manager = get_kb_manager()
|
||||
except ImportError as e:
|
||||
logger.warning(f"知识库管理器导入失败: {e}")
|
||||
_kb_checked = True
|
||||
return _kb_manager
|
||||
|
||||
|
||||
def _get_sync_service() -> Optional[Any]:
|
||||
"""
|
||||
获取同步服务实例
|
||||
|
||||
Returns:
|
||||
同步服务实例，不可用时返回 None
|
||||
"""
|
||||
try:
|
||||
from flask import current_app
|
||||
return current_app.config.get('SYNC_SERVICE')
|
||||
except Exception as e:
|
||||
logger.debug(f"获取同步服务失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ==================== 文档管理 ====================
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>/raw', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def serve_document_file(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
返回文档原始文件（仅开发模式）
|
||||
|
||||
用于前端预览文档内容，生产环境禁用此接口。
|
||||
|
||||
Args:
|
||||
doc_path: 文档相对路径（collection/filename）
|
||||
|
||||
Returns:
|
||||
文件内容或错误响应
|
||||
|
||||
Note:
|
||||
仅在 DEV_MODE=true 时可用
|
||||
"""
|
||||
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||||
return jsonify({"error": "仅开发环境可用"}), 403
|
||||
|
||||
from config import DOCUMENTS_PATH
|
||||
from flask import send_from_directory
|
||||
|
||||
filepath = os.path.join(DOCUMENTS_PATH, doc_path)
|
||||
if not os.path.exists(filepath):
|
||||
return jsonify({"error": "文件不存在"}), 404
|
||||
|
||||
directory = os.path.dirname(filepath)
|
||||
filename = os.path.basename(filepath)
|
||||
return send_from_directory(directory, filename)
|
||||
|
||||
|
||||
@document_bp.route('/documents/upload', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def upload_document() -> Tuple[Any, int]:
|
||||
"""
|
||||
上传单个文件到知识库
|
||||
|
||||
接收文件上传，保存到指定向量库目录，并触发向量化。
|
||||
|
||||
表单参数:
|
||||
file: 文件（必需）
|
||||
collection: 目标向量库名称（必需）
|
||||
|
||||
支持的文件类型:
|
||||
- PDF (.pdf)
|
||||
- Word (.docx, .doc)
|
||||
- Excel (.xlsx)
|
||||
- 文本 (.txt)
|
||||
|
||||
文件大小限制:
|
||||
- 最大 10MB
|
||||
|
||||
Returns:
|
||||
成功: {"success": true, "data": {"file": {...}, "sync_status": "..."}}
|
||||
失败: {"error": "...", "error_code": "..."}
|
||||
|
||||
Example:
|
||||
curl -X POST http://localhost:5001/documents/upload \\
|
||||
-F "file=@report.pdf" \\
|
||||
-F "collection=public_kb"
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
# 1. 检查文件
|
||||
if 'file' not in request.files:
|
||||
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
|
||||
|
||||
file = request.files['file']
|
||||
if file.filename == '':
|
||||
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
|
||||
|
||||
# 2. 获取目标向量库
|
||||
collection = request.form.get('collection') or request.form.get('kb_name')
|
||||
if not collection:
|
||||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定目标向量库 (collection 参数)", http_status=400)
|
||||
|
||||
# 3. 文件类型验证
|
||||
ext = os.path.splitext(file.filename)[1].lower()
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
return error_response(
|
||||
"UNSUPPORTED_FORMAT",
|
||||
UNSUPPORTED_FORMAT,
|
||||
f"不支持的文件类型: {ext}，支持: pdf, docx, doc, xlsx, txt",
|
||||
http_status=400
|
||||
)
|
||||
|
||||
# 4. 文件大小验证
|
||||
file.seek(0, os.SEEK_END)
|
||||
file_size = file.tell()
|
||||
file.seek(0)
|
||||
if file_size > MAX_FILE_SIZE:
|
||||
return error_response("FILE_TOO_LARGE", FILE_TOO_LARGE, "文件大小超过限制 (最大 10MB)", http_status=400)
|
||||
|
||||
# 5. 保存文件到对应目录（目录名 = 向量库名）
|
||||
target_subdir = collection
|
||||
target_dir = os.path.join(DOCUMENTS_PATH, target_subdir)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
# 安全文件名 + 处理重名
|
||||
original_filename = file.filename
|
||||
ext = os.path.splitext(original_filename)[1].lower()
|
||||
filename = safe_filename(original_filename)
|
||||
if not filename:
|
||||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||||
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
if os.path.exists(filepath):
|
||||
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
|
||||
name, ext_part = os.path.splitext(filename)
|
||||
filename = f"{name}{timestamp}{ext_part}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
|
||||
file.save(filepath)
|
||||
|
||||
# 6. 触发向量化
|
||||
sync_status = "已保存，等待手动同步"
|
||||
sync_service = _get_sync_service()
|
||||
if sync_service:
|
||||
try:
|
||||
from knowledge.sync import DocumentChange, ChangeType
|
||||
change = DocumentChange(
|
||||
document_id=f"{target_subdir}/{filename}",
|
||||
document_name=filename,
|
||||
change_type=ChangeType.ADDED,
|
||||
old_hash=None,
|
||||
new_hash=sync_service.calculate_file_hash(filepath),
|
||||
change_time=datetime.now()
|
||||
)
|
||||
sync_service.process_change(change)
|
||||
sync_status = "已保存并添加到向量库"
|
||||
except Exception as e:
|
||||
sync_status = f"已保存，向量化失败: {str(e)}"
|
||||
|
||||
return success_response(
|
||||
data={
|
||||
"file": {
|
||||
"filename": filename,
|
||||
"collection": collection,
|
||||
"path": f"{target_subdir}/{filename}",
|
||||
"size": file_size
|
||||
},
|
||||
"sync_status": sync_status
|
||||
},
|
||||
status_code=UPLOAD_SUCCESS,
|
||||
message=f"文件上传成功，{sync_status}"
|
||||
)
|
||||
|
||||
|
||||
@document_bp.route('/documents/batch-upload', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def batch_upload_documents() -> Tuple[Any, int]:
|
||||
"""
|
||||
批量上传文件到知识库
|
||||
|
||||
支持同时上传多个文件到指定向量库目录。
|
||||
|
||||
表单参数:
|
||||
files: 文件列表（必需）
|
||||
collection: 目标向量库名称（必需）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"total": N,
|
||||
"success_count": M,
|
||||
"results": [...]
|
||||
}
|
||||
}
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
# 检查文件
|
||||
if 'files' not in request.files:
|
||||
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
|
||||
|
||||
files = request.files.getlist('files')
|
||||
if not files:
|
||||
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
|
||||
|
||||
# 获取目标向量库
|
||||
collection = request.form.get('collection') or request.form.get('kb_name')
|
||||
if not collection:
|
||||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定目标向量库 (collection 参数)", http_status=400)
|
||||
|
||||
# 确定存储目录（目录名 = 向量库名）
|
||||
target_subdir = collection
|
||||
target_dir = os.path.join(DOCUMENTS_PATH, target_subdir)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
# 批量处理
|
||||
results = []
|
||||
for file in files:
|
||||
if file.filename == '':
|
||||
continue
|
||||
|
||||
ext = os.path.splitext(file.filename)[1].lower()
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
results.append({
|
||||
"filename": file.filename,
|
||||
"status": "error",
|
||||
"message": f"不支持的文件类型: {ext}"
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
original_filename = file.filename
|
||||
ext = os.path.splitext(original_filename)[1].lower()
|
||||
filename = safe_filename(original_filename)
|
||||
if not filename:
|
||||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
|
||||
# 处理重名
|
||||
if os.path.exists(filepath):
|
||||
timestamp = datetime.now().strftime('_%Y%m%d_%H%M%S')
|
||||
name, ext_part = os.path.splitext(filename)
|
||||
filename = f"{name}{timestamp}{ext_part}"
|
||||
filepath = os.path.join(target_dir, filename)
|
||||
|
||||
file.save(filepath)
|
||||
|
||||
results.append({
|
||||
"filename": filename,
|
||||
"status": "success",
|
||||
"path": f"{target_subdir}/{filename}"
|
||||
})
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": file.filename,
|
||||
"status": "error",
|
||||
"message": str(e)
|
||||
})
|
||||
|
||||
return success_response(
|
||||
data={
|
||||
"total": len(results),
|
||||
"success_count": len([r for r in results if r["status"] == "success"]),
|
||||
"results": results
|
||||
},
|
||||
status_code=BATCH_UPLOAD_SUCCESS,
|
||||
message=f"批量上传完成，成功 {len([r for r in results if r['status'] == 'success'])}/{len(results)} 个文件"
|
||||
)
|
||||
|
||||
|
||||
@document_bp.route('/documents/list', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def list_documents() -> Tuple[Any, int]:
|
||||
"""
|
||||
获取文档列表
|
||||
|
||||
扫描文档目录，返回所有可用的文档信息。
|
||||
|
||||
查询参数:
|
||||
collection: 过滤指定向量库（可选）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"documents": [
|
||||
{
|
||||
"filename": "...",
|
||||
"collection": "...",
|
||||
"path": "...",
|
||||
"size": N,
|
||||
"last_modified": "ISO 8601"
|
||||
},
|
||||
...
|
||||
],
|
||||
"total": N
|
||||
}
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
collection = request.args.get('collection') or request.args.get('kb_name')
|
||||
|
||||
# 确定要扫描的目录（目录名 = 向量库名）
|
||||
if collection:
|
||||
subdirs = [collection]
|
||||
else:
|
||||
# 列出所有文档目录
|
||||
subdirs = []
|
||||
if os.path.exists(DOCUMENTS_PATH):
|
||||
for d in os.listdir(DOCUMENTS_PATH):
|
||||
if os.path.isdir(os.path.join(DOCUMENTS_PATH, d)):
|
||||
subdirs.append(d)
|
||||
|
||||
documents = []
|
||||
supported_extensions = {'.pdf', '.docx', '.doc', '.xlsx', '.txt'}
|
||||
|
||||
for subdir in subdirs:
|
||||
level_dir = os.path.join(DOCUMENTS_PATH, subdir)
|
||||
if not os.path.exists(level_dir):
|
||||
continue
|
||||
|
||||
# 目录名即向量库名
|
||||
coll_name = subdir
|
||||
|
||||
for filename in os.listdir(level_dir):
|
||||
ext = os.path.splitext(filename)[1].lower()
|
||||
if ext not in supported_extensions:
|
||||
continue
|
||||
|
||||
filepath = os.path.join(level_dir, filename)
|
||||
try:
|
||||
stat = os.stat(filepath)
|
||||
documents.append({
|
||||
"filename": filename,
|
||||
"collection": coll_name,
|
||||
"path": f"{subdir}/{filename}",
|
||||
"size": stat.st_size,
|
||||
"last_modified": datetime.fromtimestamp(stat.st_mtime).isoformat()
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"读取文件信息失败: {filename}, {e}")
|
||||
|
||||
# 按修改时间倒序
|
||||
documents.sort(key=lambda x: x['last_modified'], reverse=True)
|
||||
|
||||
return jsonify({
|
||||
"documents": documents,
|
||||
"total": len(documents)
|
||||
})
|
||||
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>/status', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def get_document_status(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
获取文件处理状态
|
||||
|
||||
查询指定文档的向量化状态和切片数量。
|
||||
|
||||
Args:
|
||||
doc_path: 文档相对路径（collection/filename）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"status": "processed|pending|error",
|
||||
"chunk_count": N,
|
||||
"last_processed": "ISO 8601"
|
||||
}
|
||||
"""
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
# 解析路径
|
||||
parts = doc_path.split('/')
|
||||
if len(parts) < 2:
|
||||
return jsonify({"error": "无效的文档路径"}), 400
|
||||
|
||||
subdir = parts[0]
|
||||
filename = '/'.join(parts[1:])
|
||||
|
||||
# 目录名即向量库名
|
||||
collection = subdir
|
||||
|
||||
# 获取文档信息
|
||||
from config import DOCUMENTS_PATH
|
||||
file_on_disk = os.path.isfile(os.path.join(DOCUMENTS_PATH, subdir, filename))
|
||||
|
||||
doc_info = kb_manager.get_document_info(collection, filename)
|
||||
|
||||
if not doc_info:
|
||||
if file_on_disk:
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"status": "unprocessed",
|
||||
"chunk_count": 0,
|
||||
"last_processed": None
|
||||
})
|
||||
return jsonify({"error": "文档不存在"}), 404
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"status": doc_info.get("status", "unknown"),
|
||||
"chunk_count": doc_info.get("total_chunks", 0),
|
||||
"last_processed": doc_info.get("effective_date") or doc_info.get("version")
|
||||
})
|
||||
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>', methods=['PUT'])
|
||||
@require_gateway_auth
|
||||
def update_document(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
更新文件（重新上传覆盖）
|
||||
|
||||
替换现有文件内容，并触发重新向量化。
|
||||
|
||||
Args:
|
||||
doc_path: 文档相对路径（collection/filename）
|
||||
|
||||
表单参数:
|
||||
file: 新文件（必需）
|
||||
|
||||
Returns:
|
||||
{"success": true, "message": "文件已更新"}
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
if 'file' not in request.files:
|
||||
return jsonify({"error": "没有上传文件"}), 400
|
||||
|
||||
file = request.files['file']
|
||||
if file.filename == '':
|
||||
return jsonify({"error": "没有选择文件"}), 400
|
||||
|
||||
# 解析路径
|
||||
parts = doc_path.split('/')
|
||||
if len(parts) < 2:
|
||||
return jsonify({"error": "无效的文档路径"}), 400
|
||||
|
||||
filepath = os.path.join(DOCUMENTS_PATH, doc_path)
|
||||
if not os.path.exists(filepath):
|
||||
return jsonify({"error": "文件不存在"}), 404
|
||||
|
||||
# 覆盖文件
|
||||
file.save(filepath)
|
||||
|
||||
# 触发重新向量化
|
||||
sync_service = _get_sync_service()
|
||||
if sync_service:
|
||||
try:
|
||||
subdir = parts[0]
|
||||
filename = '/'.join(parts[1:])
|
||||
from knowledge.sync import DocumentChange, ChangeType
|
||||
change = DocumentChange(
|
||||
document_id=doc_path,
|
||||
document_name=filename,
|
||||
change_type=ChangeType.MODIFIED,
|
||||
old_hash=None,
|
||||
new_hash=sync_service.calculate_file_hash(filepath),
|
||||
change_time=datetime.now()
|
||||
)
|
||||
sync_service.process_change(change)
|
||||
except Exception as e:
|
||||
logger.warning(f"重新向量化失败: {e}")
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"message": "文件已更新"
|
||||
})
|
||||
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>', methods=['DELETE'])
|
||||
@require_gateway_auth
|
||||
def delete_document(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
删除文档
|
||||
|
||||
同时删除向量库中的切片和物理文件。
|
||||
|
||||
Args:
|
||||
doc_path: 文档相对路径（collection/filename）
|
||||
|
||||
Returns:
|
||||
{"success": true, "message": "文档已删除"}
|
||||
|
||||
Note:
|
||||
此操作不可逆，请谨慎使用
|
||||
"""
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
# 解析路径
|
||||
parts = doc_path.split('/')
|
||||
if len(parts) < 2:
|
||||
return jsonify({"error": "无效的文档路径"}), 400
|
||||
|
||||
subdir = parts[0]
|
||||
filename = '/'.join(parts[1:])
|
||||
|
||||
# 目录名即向量库名
|
||||
collection = subdir
|
||||
|
||||
filepath = os.path.join(DOCUMENTS_PATH, doc_path)
|
||||
if not os.path.exists(filepath):
|
||||
return jsonify({"error": "文件不存在"}), 404
|
||||
|
||||
try:
|
||||
# 1. 从向量库删除（source 存的是文件名，不是完整路径）
|
||||
kb_manager = _get_kb_manager()
|
||||
if kb_manager:
|
||||
kb_manager.delete_document(collection, filename)
|
||||
|
||||
# 2. 删除文件
|
||||
os.remove(filepath)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"message": "文档已删除"
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": f"删除失败: {str(e)}"}), 500
|
||||
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>/chunks', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
查看文件切片
|
||||
|
||||
返回指定文档的所有切片内容，用于调试和验证。
|
||||
|
||||
Args:
|
||||
doc_path: 文档相对路径（collection/filename）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"document_id": "...",
|
||||
"collection": "...",
|
||||
"chunks": [...],
|
||||
"total": N
|
||||
}
|
||||
"""
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
# 解析路径
|
||||
parts = doc_path.split('/')
|
||||
if len(parts) < 2:
|
||||
return jsonify({"error": "无效的文档路径"}), 400
|
||||
|
||||
subdir = parts[0]
|
||||
# 目录名即向量库名
|
||||
collection = subdir
|
||||
|
||||
chunks = kb_manager.get_document_chunks(collection, os.path.basename(doc_path))
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"document_id": doc_path,
|
||||
"collection": collection,
|
||||
"chunks": chunks,
|
||||
"total": len(chunks)
|
||||
})
|
||||
|
||||
|
||||
@document_bp.route('/documents/<path:doc_path>/preview', methods=['GET'])
|
||||
@require_gateway_auth
|
||||
def preview_document(doc_path: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
文档预览接口（支持按切片序号跳转）
|
||||
|
||||
用于前端引用溯源点击跳转：给定 chunk_index，返回目标切片及其上下文。
|
||||
复用现有切片查询逻辑，不新增存储。
|
||||
|
||||
Query Params:
|
||||
chunk_index (int): 目标切片序号（来自 citation 的 chunk_index 字段）
|
||||
context (int): 上下文切片数，默认 2（前后各取 2 个）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"success": true,
|
||||
"collection": "...",
|
||||
"source": "文件名",
|
||||
"total_chunks": N,
|
||||
"target_index": M,
|
||||
"chunks": [
|
||||
{"id": "...", "content": "...", "metadata": {...}, "is_target": true/false},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
# 解析路径
|
||||
parts = doc_path.split('/')
|
||||
if len(parts) < 2:
|
||||
return jsonify({"error": "无效的文档路径，格式: collection/filename"}), 400
|
||||
|
||||
collection = parts[0]
|
||||
filename = os.path.basename(doc_path)
|
||||
|
||||
# 查询参数
|
||||
chunk_index_str = request.args.get('chunk_index')
|
||||
context_count = int(request.args.get('context', 2))
|
||||
|
||||
# 获取所有切片
|
||||
all_chunks = kb_manager.get_document_chunks(collection, filename)
|
||||
if not all_chunks:
|
||||
return jsonify({"error": f"文档 '{filename}' 不存在或无切片"}), 404
|
||||
|
||||
total = len(all_chunks)
|
||||
|
||||
# 如果未指定 chunk_index，返回前 5 个切片作为概览
|
||||
if chunk_index_str is None:
|
||||
preview_chunks = all_chunks[:5]
|
||||
for c in preview_chunks:
|
||||
c['is_target'] = False
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"collection": collection,
|
||||
"source": filename,
|
||||
"total_chunks": total,
|
||||
"target_index": None,
|
||||
"chunks": preview_chunks
|
||||
})
|
||||
|
||||
# 定位目标切片 —— 按 meta.chunk_index 排序后查找，避免数组下标与 chunk_index 不一致
|
||||
try:
|
||||
target_chunk_index = int(chunk_index_str)
|
||||
except ValueError:
|
||||
return jsonify({"error": "chunk_index 必须为整数"}), 400
|
||||
|
||||
# 按 chunk_index 排序（Chroma 返回顺序不保证有序）
|
||||
all_chunks.sort(key=lambda c: c.get('metadata', {}).get('chunk_index', 0))
|
||||
|
||||
# 按 meta.chunk_index 查找目标切片在排序后数组中的实际位置
|
||||
target_pos = None
|
||||
for i, c in enumerate(all_chunks):
|
||||
if c.get('metadata', {}).get('chunk_index') == target_chunk_index:
|
||||
target_pos = i
|
||||
break
|
||||
|
||||
# 回退：直接用数组下标（兼容旧数据无 chunk_index 字段的情况）
|
||||
if target_pos is None:
|
||||
if 0 <= target_chunk_index < total:
|
||||
target_pos = target_chunk_index
|
||||
else:
|
||||
max_idx = max(
|
||||
(c.get('metadata', {}).get('chunk_index', 0) for c in all_chunks),
|
||||
default=total - 1
|
||||
)
|
||||
return jsonify({
|
||||
"error": f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})"
|
||||
}), 400
|
||||
|
||||
# 截取上下文窗口
|
||||
start = max(0, target_pos - context_count)
|
||||
end = min(total, target_pos + context_count + 1)
|
||||
window = all_chunks[start:end]
|
||||
|
||||
# 标记目标切片
|
||||
for i, c in enumerate(window):
|
||||
c['is_target'] = (start + i == target_pos)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"collection": collection,
|
||||
"source": filename,
|
||||
"total_chunks": total,
|
||||
"target_index": target_chunk_index,
|
||||
"chunks": window
|
||||
})
|
||||
|
||||
|
||||
# ==================== 切片管理 ====================
|
||||
|
||||
@document_bp.route('/chunks', methods=['POST'])
|
||||
@require_gateway_auth
|
||||
def create_chunk() -> Tuple[Any, int]:
|
||||
"""
|
||||
新增切片
|
||||
|
||||
手动向向量库添加一个切片，用于补充或修正内容。
|
||||
|
||||
请求体:
|
||||
{
|
||||
"collection": "向量库名称",
|
||||
"content": "切片内容",
|
||||
"metadata": {} // 可选
|
||||
}
|
||||
|
||||
Returns:
|
||||
{"success": true, "chunk_id": "...", "message": "切片已添加"}
|
||||
"""
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
data = request.json or {}
|
||||
collection = data.get('collection')
|
||||
content = data.get('content')
|
||||
metadata = data.get('metadata', {})
|
||||
|
||||
if not collection:
|
||||
return jsonify({"error": "请指定向量库 (collection)"}), 400
|
||||
if not content:
|
||||
return jsonify({"error": "切片内容不能为空"}), 400
|
||||
|
||||
chunk_id = kb_manager.add_chunk(collection, content, metadata)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"chunk_id": chunk_id,
|
||||
"message": "切片已添加"
|
||||
})
|
||||
|
||||
|
||||
@document_bp.route('/chunks/<chunk_id>', methods=['PUT'])
|
||||
@require_gateway_auth
|
||||
def update_chunk(chunk_id: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
修改切片
|
||||
|
||||
更新指定切片的内容或元数据。
|
||||
|
||||
Args:
|
||||
chunk_id: 切片 ID
|
||||
|
||||
请求体:
|
||||
{
|
||||
"collection": "向量库名称",
|
||||
"content": "新内容", // 可选
|
||||
"metadata": {} // 可选
|
||||
}
|
||||
|
||||
Returns:
|
||||
{"success": true, "message": "切片已更新"}
|
||||
"""
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
data = request.json or {}
|
||||
collection = data.get('collection')
|
||||
content = data.get('content')
|
||||
metadata = data.get('metadata')
|
||||
|
||||
if not collection:
|
||||
return jsonify({"error": "请指定向量库 (collection)"}), 400
|
||||
|
||||
success = kb_manager.update_chunk(collection, chunk_id, content=content, metadata=metadata)
|
||||
|
||||
if success:
|
||||
return jsonify({"success": True, "message": "切片已更新"})
|
||||
return jsonify({"error": "更新失败"}), 500
|
||||
|
||||
|
||||
@document_bp.route('/chunks/<chunk_id>', methods=['DELETE'])
|
||||
@require_gateway_auth
|
||||
def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
|
||||
"""
|
||||
删除切片
|
||||
|
||||
从向量库中移除指定切片。如果删除后该文件没有其他切片，
|
||||
同时清理文档哈希记录，以便同步服务能重新检测该文件。
|
||||
|
||||
Args:
|
||||
chunk_id: 切片 ID
|
||||
|
||||
请求体或查询参数:
|
||||
collection: 向量库名称（必需）
|
||||
|
||||
Returns:
|
||||
{"success": true, "message": "切片已删除"}
|
||||
"""
|
||||
data = request.get_json(silent=True) or {}
|
||||
collection = data.get('collection') if data else None
|
||||
|
||||
if not collection:
|
||||
collection = request.args.get('collection')
|
||||
|
||||
if not collection:
|
||||
return jsonify({"error": "请指定向量库 (collection)"}), 400
|
||||
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
# 删除切片，返回 (success, source_file)
|
||||
success, source_file = kb_manager.delete_chunk(collection, chunk_id)
|
||||
|
||||
if success and source_file:
|
||||
# 检查该文件是否还有其他切片
|
||||
remaining_chunks = kb_manager.list_chunks(collection, limit=100000)
|
||||
has_other_chunks = any(
|
||||
c.get('metadata', {}).get('source') == source_file
|
||||
for c in remaining_chunks
|
||||
)
|
||||
|
||||
# 如果没有其他切片了，删除哈希记录
|
||||
if not has_other_chunks:
|
||||
try:
|
||||
from knowledge.sync import SyncDatabase
|
||||
sync_db = SyncDatabase()
|
||||
document_id = f"{collection}/{source_file}"
|
||||
sync_db.delete_document_hash(document_id)
|
||||
logger.info(f"清理文档哈希记录: {document_id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理哈希记录失败: {e}")
|
||||
|
||||
if success:
|
||||
return jsonify({"success": True, "message": "切片已删除"})
|
||||
return jsonify({"error": "删除失败"}), 500
|
||||
|
||||
|
||||
@document_bp.route('/chunks/batch', methods=['DELETE'])
|
||||
@require_gateway_auth
|
||||
def delete_chunks_by_source() -> Tuple[Any, int]:
|
||||
"""
|
||||
批量删除指定文件的所有切片
|
||||
|
||||
删除指定向量库中某个文件的所有切片，并清理哈希记录。
|
||||
相比逐个删除，批量删除更高效，避免前端超时。
|
||||
|
||||
请求体:
|
||||
{
|
||||
"collection": "向量库名称",
|
||||
"source": "文件名"
|
||||
}
|
||||
|
||||
Returns:
|
||||
{"success": true, "deleted_count": N, "message": "..."}
|
||||
"""
|
||||
data = request.get_json(silent=True) or {}
|
||||
collection = data.get('collection')
|
||||
source = data.get('source')
|
||||
|
||||
if not collection:
|
||||
return jsonify({"error": "请指定向量库 (collection)"}), 400
|
||||
if not source:
|
||||
return jsonify({"error": "请指定文件名 (source)"}), 400
|
||||
|
||||
kb_manager = _get_kb_manager()
|
||||
if not kb_manager:
|
||||
return jsonify({"error": "知识库管理器未初始化"}), 503
|
||||
|
||||
# 批量删除该文件的所有切片
|
||||
try:
|
||||
deleted_count = kb_manager.delete_chunks_by_source(collection, source)
|
||||
|
||||
# 清理哈希记录
|
||||
if deleted_count > 0:
|
||||
try:
|
||||
from knowledge.sync import SyncDatabase
|
||||
sync_db = SyncDatabase()
|
||||
# 统一使用正斜杠格式（与 sync.py scan_documents 一致）
|
||||
document_id = f"{collection}/{source}"
|
||||
sync_db.delete_document_hash(document_id)
|
||||
logger.info(f"清理文档哈希记录: {document_id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理哈希记录失败: {e}")
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"deleted_count": deleted_count,
|
||||
"message": f"已删除 {deleted_count} 个切片"
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"批量删除切片失败: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
Reference in New Issue
Block a user