183a57e7f1
- 将全部路由文件（12个）的 jsonify 响应迁移至 success_response/error_response 统一格式 - 修复 sync_routes.py error_response 参数错误（P0） - 新增异步任务系统：task_registry + task_routes - 新增状态码：TASK_NOT_FOUND(4014)、TASK_CONFLICT(4015)、REINDEX_ERROR(5040) - 修正 task_routes/exam_pkg 中语义不匹配的状态码 - 更新 curl 测试手册、后端对接规范文档 - 添加缓存性能报告和 Redis 迁移计划
1180 lines
40 KiB
Python
1180 lines
40 KiB
Python
"""
|
||
文档管理 API
|
||
|
||
本模块提供文档上传、查询、删除等管理功能，包括：
|
||
- 单文件和批量上传
|
||
- 文档列表查询
|
||
- 文档切片管理
|
||
|
||
路由列表:
|
||
POST /documents/upload : 上传文件（单个）
|
||
POST /documents/batch-upload : 批量上传文件
|
||
GET /documents/list : 文档列表
|
||
GET /documents/<doc_id>/status : 文件处理状态
|
||
PUT /documents/<doc_id> : 更新文件
|
||
DELETE /documents/<doc_path> : 删除文档
|
||
GET /documents/<doc_id>/chunks : 查看文件切片
|
||
GET /documents/<doc_id>/preview : 文档预览（支持按切片序号跳转）
|
||
|
||
切片管理:
|
||
POST /chunks : 新增切片
|
||
PUT /chunks/<chunk_id> : 修改切片
|
||
DELETE /chunks/<chunk_id> : 删除切片
|
||
|
||
架构说明:
|
||
- 权限验证由后端网关完成，RAG 服务不做权限判断
|
||
- 文档存储在 documents/<collection>/ 目录下
|
||
- 上传后自动触发向量化（如果同步服务可用）
|
||
|
||
Example:
|
||
# 上传文件
|
||
curl -X POST http://localhost:5001/documents/upload \\
|
||
-H "Authorization: Bearer mock-token-admin" \\
|
||
-F "file=@report.pdf" \\
|
||
-F "collection=public_kb"
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import uuid
|
||
from datetime import datetime
|
||
from typing import Optional, Tuple, Any, List, Dict
|
||
from flask import Blueprint, request
|
||
import logging
|
||
|
||
logger = logging.getLogger(__name__)
|
||
from werkzeug.utils import secure_filename
|
||
from auth.gateway import require_gateway_auth
|
||
from core.status_codes import (
|
||
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST, SUCCESS,
|
||
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION, NOT_FOUND,
|
||
FILE_TOO_LARGE, UNSUPPORTED_FORMAT, INTERNAL_ERROR,
|
||
SERVICE_UNAVAILABLE, DELETE_SUCCESS, UPDATE_SUCCESS, NO_CONTENT,
|
||
FORBIDDEN
|
||
)
|
||
from api.response_utils import success_response, error_response
|
||
|
||
document_bp = Blueprint('document', __name__)
|
||
|
||
# 文件限制
|
||
ALLOWED_EXTENSIONS = {'.pdf', '.docx', '.doc', '.xlsx', '.txt'}
|
||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
||
|
||
|
||
def safe_filename(filename: str) -> str:
|
||
"""
|
||
安全文件名处理 - 保留中文，仅移除危险字符
|
||
|
||
与 werkzeug.secure_filename() 不同，此函数保留 Unicode 字符（包括中文），
|
||
仅移除路径分隔符和其他可能导致路径遍历攻击的字符。
|
||
|
||
Args:
|
||
filename: 原始文件名
|
||
|
||
Returns:
|
||
安全的文件名；无效输入返回空字符串
|
||
|
||
Example:
|
||
>>> safe_filename("财务报告.pdf")
|
||
'财务报告.pdf'
|
||
>>> safe_filename("../../../etc/passwd")
|
||
'etc_passwd'
|
||
"""
|
||
if not filename:
|
||
return ""
|
||
|
||
# 移除路径分隔符和危险字符
|
||
dangerous_chars = ['/', '\\', '..', '\x00', '\n', '\r', '\t']
|
||
safe_name = filename
|
||
for char in dangerous_chars:
|
||
safe_name = safe_name.replace(char, '_')
|
||
|
||
# 移除首尾空格和点
|
||
safe_name = safe_name.strip(' .')
|
||
|
||
# 如果文件名为空或只有扩展名，返回空
|
||
if not safe_name or safe_name.startswith('.') and safe_name.count('.') == 1:
|
||
return ""
|
||
|
||
return safe_name
|
||
|
||
|
||
def _validate_doc_path(doc_path: str, documents_path: str) -> str:
|
||
"""
|
||
校验文档路径是否在允许目录内，防止路径遍历攻击。
|
||
|
||
Returns:
|
||
解析后的安全绝对路径
|
||
Raises:
|
||
ValueError: 路径不合法
|
||
"""
|
||
filepath = os.path.join(documents_path, doc_path)
|
||
real_path = os.path.realpath(filepath)
|
||
real_base = os.path.realpath(documents_path)
|
||
if not real_path.startswith(real_base + os.sep) and real_path != real_base:
|
||
raise ValueError("非法路径")
|
||
return real_path
|
||
|
||
|
||
# 延迟初始化缓存
|
||
_kb_manager = None
|
||
_kb_checked = False
|
||
|
||
|
||
def _get_kb_manager() -> Optional[Any]:
|
||
"""
|
||
获取知识库管理器（延迟加载）
|
||
|
||
Returns:
|
||
知识库管理器实例，导入失败返回 None
|
||
"""
|
||
global _kb_manager, _kb_checked
|
||
if not _kb_checked:
|
||
try:
|
||
from knowledge.manager import get_kb_manager
|
||
_kb_manager = get_kb_manager()
|
||
except ImportError as e:
|
||
logger.warning(f"知识库管理器导入失败: {e}")
|
||
_kb_checked = True
|
||
return _kb_manager
|
||
|
||
|
||
def _get_sync_service() -> Optional[Any]:
|
||
"""
|
||
获取同步服务实例
|
||
|
||
Returns:
|
||
同步服务实例，不可用时返回 None
|
||
"""
|
||
try:
|
||
from flask import current_app
|
||
return current_app.config.get('SYNC_SERVICE')
|
||
except Exception as e:
|
||
logger.debug(f"获取同步服务失败: {e}")
|
||
return None
|
||
|
||
|
||
# ==================== 文档管理 ====================
|
||
|
||
@document_bp.route('/documents/<path:doc_path>/raw', methods=['GET'])
|
||
@require_gateway_auth
|
||
def serve_document_file(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
返回文档原始文件（仅开发模式）
|
||
|
||
用于前端预览文档内容，生产环境禁用此接口。
|
||
|
||
Args:
|
||
doc_path: 文档相对路径（collection/filename）
|
||
|
||
Returns:
|
||
文件内容或错误响应
|
||
|
||
Note:
|
||
仅在 DEV_MODE=true 时可用
|
||
"""
|
||
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
|
||
|
||
from config import DOCUMENTS_PATH
|
||
from flask import send_from_directory
|
||
|
||
filepath = os.path.join(DOCUMENTS_PATH, doc_path)
|
||
try:
|
||
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
|
||
except ValueError:
|
||
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
|
||
if not os.path.exists(filepath):
|
||
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
|
||
|
||
directory = os.path.dirname(filepath)
|
||
filename = os.path.basename(filepath)
|
||
return send_from_directory(directory, filename)
|
||
|
||
|
||
@document_bp.route('/documents/upload', methods=['POST'])
|
||
@require_gateway_auth
|
||
def upload_document() -> Tuple[Any, int]:
|
||
"""
|
||
上传单个文件到知识库
|
||
|
||
接收文件上传，保存到指定向量库目录，并触发向量化。
|
||
|
||
表单参数:
|
||
file: 文件（必需）
|
||
collection: 目标向量库名称（必需）
|
||
|
||
支持的文件类型:
|
||
- PDF (.pdf)
|
||
- Word (.docx, .doc)
|
||
- Excel (.xlsx)
|
||
- 文本 (.txt)
|
||
|
||
文件大小限制:
|
||
- 最大 10MB
|
||
|
||
Returns:
|
||
成功: {"success": true, "data": {"file": {...}, "sync_status": "..."}}
|
||
失败: {"error": "...", "error_code": "..."}
|
||
|
||
Example:
|
||
curl -X POST http://localhost:5001/documents/upload \\
|
||
-F "file=@report.pdf" \\
|
||
-F "collection=public_kb"
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
|
||
# 1. 检查文件
|
||
if 'file' not in request.files:
|
||
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
|
||
|
||
file = request.files['file']
|
||
if file.filename == '':
|
||
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
|
||
|
||
# 2. 获取目标向量库
|
||
collection = request.form.get('collection') or request.form.get('kb_name')
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定目标向量库 (collection 参数)", http_status=400)
|
||
|
||
# 3. 文件类型验证
|
||
ext = os.path.splitext(file.filename)[1].lower()
|
||
if ext not in ALLOWED_EXTENSIONS:
|
||
return error_response(
|
||
"UNSUPPORTED_FORMAT",
|
||
UNSUPPORTED_FORMAT,
|
||
f"不支持的文件类型: {ext}，支持: pdf, docx, doc, xlsx, txt",
|
||
http_status=400
|
||
)
|
||
|
||
# 4. 文件大小验证
|
||
file.seek(0, os.SEEK_END)
|
||
file_size = file.tell()
|
||
file.seek(0)
|
||
if file_size > MAX_FILE_SIZE:
|
||
return error_response("FILE_TOO_LARGE", FILE_TOO_LARGE, "文件大小超过限制 (最大 10MB)", http_status=400)
|
||
|
||
# 5. 保存文件到对应目录（目录名 = 向量库名）
|
||
target_subdir = collection
|
||
target_dir = os.path.join(DOCUMENTS_PATH, target_subdir)
|
||
os.makedirs(target_dir, exist_ok=True)
|
||
|
||
# 安全文件名 + 处理重名
|
||
original_filename = file.filename
|
||
ext = os.path.splitext(original_filename)[1].lower()
|
||
filename = safe_filename(original_filename)
|
||
if not filename:
|
||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||
|
||
filepath = os.path.join(target_dir, filename)
|
||
|
||
# 同名文件处理：清理旧切片 + 覆盖旧文件（替代原来的时间戳重命名策略）
|
||
replaced = False
|
||
if os.path.exists(filepath):
|
||
replaced = True
|
||
# 1. 清理旧切片（ChromaDB + BM25 + DocStore）
|
||
kb_manager = _get_kb_manager()
|
||
if kb_manager:
|
||
old_chunks = kb_manager.get_document_chunks(collection, filename)
|
||
if old_chunks:
|
||
old_ids = [c['id'] for c in old_chunks]
|
||
coll_obj = kb_manager.get_collection(collection)
|
||
if coll_obj:
|
||
coll_obj.delete(ids=old_ids)
|
||
logger.info(f"替换上传: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
|
||
|
||
# 清理关联的 DocStore 文件
|
||
try:
|
||
from pathlib import Path as _Path
|
||
docstore_dir = _Path('.data/docstore')
|
||
if docstore_dir.exists():
|
||
for ds_file in docstore_dir.glob(f'{collection}_{filename}_*.json'):
|
||
ds_file.unlink()
|
||
except Exception as e:
|
||
logger.warning(f"清理 DocStore 失败: {e}")
|
||
|
||
# 重建 BM25 索引
|
||
kb_manager.rebuild_bm25_index(collection)
|
||
|
||
# 2. 清理同步哈希记录（以便 sync 能正确检测为新文件）
|
||
try:
|
||
from knowledge.sync import SyncDatabase
|
||
sync_db = SyncDatabase()
|
||
sync_db.delete_document_hash(f"{collection}/{filename}")
|
||
except Exception as e:
|
||
logger.warning(f"清理哈希记录失败: {e}")
|
||
|
||
# 保存文件（覆盖同名旧文件）
|
||
file.save(filepath)
|
||
|
||
# 版本管理：覆盖上传时标记旧版本为 superseded（新版本记录由 sync 服务统一创建）
|
||
if replaced:
|
||
try:
|
||
from knowledge.document_versions import get_version_query
|
||
from data.db import get_connection
|
||
vq = get_version_query()
|
||
active = vq.get_active_version(collection, filename)
|
||
|
||
if active:
|
||
with get_connection("knowledge") as conn:
|
||
conn.execute("""
|
||
UPDATE document_versions
|
||
SET status='superseded', deprecated_date=?,
|
||
deprecated_reason='重新上传覆盖'
|
||
WHERE collection=? AND document_id=? AND version=?
|
||
""", (datetime.now().isoformat(), collection,
|
||
filename, active.version))
|
||
conn.commit()
|
||
vq.log_version_change(
|
||
collection, filename,
|
||
change_type="reupload",
|
||
old_version=active.version,
|
||
old_status="active", new_status="superseded",
|
||
reason="重新上传覆盖",
|
||
changed_by=request.current_user.get('user_id', '')
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"标记旧版本失败: {e}")
|
||
|
||
# 6. 触发向量化（异步任务）
|
||
sync_status = "已保存，等待手动同步"
|
||
sync_service = _get_sync_service()
|
||
task_id = None
|
||
|
||
if sync_service:
|
||
from core.task_registry import get_registry
|
||
registry = get_registry()
|
||
|
||
task = registry.create_task('upload', f"向量化: {filename}")
|
||
task_id = task.id
|
||
|
||
def _do_vectorize(task, svc, change_obj, fname):
|
||
"""后台执行向量化"""
|
||
registry.update_progress(task.id, stage='向量化', message=f"正在处理: {fname}")
|
||
svc.process_change(change_obj)
|
||
return {'filename': fname, 'sync_status': '已添加到向量库'}
|
||
|
||
try:
|
||
from knowledge.sync import DocumentChange, ChangeType
|
||
change = DocumentChange(
|
||
document_id=f"{target_subdir}/{filename}",
|
||
document_name=filename,
|
||
change_type=ChangeType.ADDED,
|
||
old_hash=None,
|
||
new_hash=sync_service.calculate_file_hash(filepath),
|
||
change_time=datetime.now()
|
||
)
|
||
registry.start_task(task.id, _do_vectorize, sync_service, change, filename)
|
||
sync_status = "已保存，向量化任务已启动"
|
||
except Exception as e:
|
||
logger.warning(f"创建向量化任务失败: {e}")
|
||
registry.fail_task(task.id, str(e))
|
||
sync_status = "已保存，向量化任务创建失败"
|
||
|
||
return success_response(
|
||
data={
|
||
"file": {
|
||
"filename": filename,
|
||
"collection": collection,
|
||
"path": f"{target_subdir}/{filename}",
|
||
"size": file_size,
|
||
"replaced": replaced
|
||
},
|
||
"sync_status": sync_status,
|
||
"task_id": task_id
|
||
},
|
||
status_code=UPLOAD_SUCCESS,
|
||
message=f"文件上传成功，{sync_status}"
|
||
)
|
||
|
||
|
||
@document_bp.route('/documents/batch-upload', methods=['POST'])
|
||
@require_gateway_auth
|
||
def batch_upload_documents() -> Tuple[Any, int]:
|
||
"""
|
||
批量上传文件到知识库
|
||
|
||
支持同时上传多个文件到指定向量库目录。
|
||
|
||
表单参数:
|
||
files: 文件列表（必需）
|
||
collection: 目标向量库名称（必需）
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"data": {
|
||
"total": N,
|
||
"success_count": M,
|
||
"results": [...]
|
||
}
|
||
}
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
|
||
# 检查文件
|
||
if 'files' not in request.files:
|
||
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
|
||
|
||
files = request.files.getlist('files')
|
||
if not files:
|
||
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
|
||
|
||
# 获取目标向量库
|
||
collection = request.form.get('collection') or request.form.get('kb_name')
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定目标向量库 (collection 参数)", http_status=400)
|
||
|
||
# 确定存储目录（目录名 = 向量库名）
|
||
target_subdir = collection
|
||
target_dir = os.path.join(DOCUMENTS_PATH, target_subdir)
|
||
os.makedirs(target_dir, exist_ok=True)
|
||
|
||
# 批量处理
|
||
results = []
|
||
for file in files:
|
||
if file.filename == '':
|
||
continue
|
||
|
||
ext = os.path.splitext(file.filename)[1].lower()
|
||
if ext not in ALLOWED_EXTENSIONS:
|
||
results.append({
|
||
"filename": file.filename,
|
||
"status": "error",
|
||
"message": f"不支持的文件类型: {ext}"
|
||
})
|
||
continue
|
||
|
||
# 文件大小校验
|
||
file.seek(0, 2)
|
||
file_size = file.tell()
|
||
file.seek(0)
|
||
if file_size > MAX_FILE_SIZE:
|
||
results.append({
|
||
"filename": file.filename,
|
||
"status": "error",
|
||
"message": f"文件过大（最大 {MAX_FILE_SIZE // 1024 // 1024}MB）"
|
||
})
|
||
continue
|
||
|
||
try:
|
||
original_filename = file.filename
|
||
ext = os.path.splitext(original_filename)[1].lower()
|
||
filename = safe_filename(original_filename)
|
||
if not filename:
|
||
filename = f"upload_{datetime.now().strftime('%Y%m%d%H%M%S')}{ext}"
|
||
filepath = os.path.join(target_dir, filename)
|
||
|
||
# 同名文件处理：清理旧切片 + 覆盖旧文件
|
||
replaced = False
|
||
if os.path.exists(filepath):
|
||
replaced = True
|
||
kb_mgr = _get_kb_manager()
|
||
if kb_mgr:
|
||
old_chunks = kb_mgr.get_document_chunks(collection, filename)
|
||
if old_chunks:
|
||
old_ids = [c['id'] for c in old_chunks]
|
||
coll_obj = kb_mgr.get_collection(collection)
|
||
if coll_obj:
|
||
coll_obj.delete(ids=old_ids)
|
||
logger.info(f"批量替换: 清理旧切片 {filename} -> {collection}, 共 {len(old_ids)} 个")
|
||
kb_mgr.rebuild_bm25_index(collection)
|
||
|
||
try:
|
||
from knowledge.sync import SyncDatabase
|
||
sync_db = SyncDatabase()
|
||
sync_db.delete_document_hash(f"{collection}/{filename}")
|
||
except Exception as e:
|
||
logger.warning(f"清理哈希记录失败: {e}")
|
||
|
||
file.save(filepath)
|
||
|
||
# 版本管理：覆盖上传时标记旧版本为 superseded（新版本记录由 sync 服务统一创建）
|
||
if replaced:
|
||
try:
|
||
from knowledge.document_versions import get_version_query
|
||
from data.db import get_connection
|
||
vq = get_version_query()
|
||
active = vq.get_active_version(collection, filename)
|
||
|
||
if active:
|
||
with get_connection("knowledge") as conn:
|
||
conn.execute("""
|
||
UPDATE document_versions
|
||
SET status='superseded', deprecated_date=?,
|
||
deprecated_reason='批量上传覆盖'
|
||
WHERE collection=? AND document_id=? AND version=?
|
||
""", (datetime.now().isoformat(), collection,
|
||
filename, active.version))
|
||
conn.commit()
|
||
except Exception as e:
|
||
logger.warning(f"批量上传标记旧版本失败: {e}")
|
||
|
||
results.append({
|
||
"filename": filename,
|
||
"status": "success",
|
||
"path": f"{target_subdir}/{filename}",
|
||
"replaced": replaced
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"上传处理异常: {e}")
|
||
results.append({
|
||
"filename": file.filename,
|
||
"status": "error",
|
||
"message": "上传处理失败"
|
||
})
|
||
|
||
success_count = len([r for r in results if r["status"] == "success"])
|
||
task_id = None
|
||
|
||
# 批量上传完成后自动触发向量化（异步任务）
|
||
sync_service = _get_sync_service()
|
||
if sync_service and success_count > 0:
|
||
from core.task_registry import get_registry
|
||
registry = get_registry()
|
||
|
||
running = registry.list_tasks(status='running', task_type='batch_upload', limit=1)
|
||
if not running:
|
||
task = registry.create_task('batch_upload', f"批量向量化: {success_count} 个文件", total=success_count)
|
||
task_id = task.id
|
||
|
||
def _do_batch_vectorize(task, svc, count):
|
||
processed = [0]
|
||
|
||
def on_change(change):
|
||
processed[0] += 1
|
||
registry.update_progress(
|
||
task.id, current=processed[0], total=count,
|
||
stage='批量向量化',
|
||
message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}"
|
||
)
|
||
|
||
old_cb = svc.on_change_callback
|
||
svc.on_change_callback = on_change
|
||
try:
|
||
registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...')
|
||
result = svc.sync_now()
|
||
return {
|
||
'synced': result.documents_processed,
|
||
'added': result.documents_added,
|
||
'errors': result.errors,
|
||
}
|
||
finally:
|
||
svc.on_change_callback = old_cb
|
||
|
||
registry.start_task(task.id, _do_batch_vectorize, sync_service, success_count)
|
||
|
||
return success_response(
|
||
data={
|
||
"total": len(results),
|
||
"success_count": success_count,
|
||
"results": results,
|
||
"task_id": task_id
|
||
},
|
||
status_code=BATCH_UPLOAD_SUCCESS,
|
||
message=f"批量上传完成，成功 {success_count}/{len(results)} 个文件"
|
||
)
|
||
|
||
|
||
@document_bp.route('/documents/list', methods=['GET'])
|
||
@require_gateway_auth
|
||
def list_documents() -> Tuple[Any, int]:
|
||
"""
|
||
获取文档列表
|
||
|
||
扫描文档目录，返回所有可用的文档信息。
|
||
|
||
查询参数:
|
||
collection: 过滤指定向量库（可选）
|
||
|
||
Returns:
|
||
{
|
||
"documents": [
|
||
{
|
||
"filename": "...",
|
||
"collection": "...",
|
||
"path": "...",
|
||
"size": N,
|
||
"last_modified": "ISO 8601"
|
||
},
|
||
...
|
||
],
|
||
"total": N
|
||
}
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
|
||
collection = request.args.get('collection') or request.args.get('kb_name')
|
||
|
||
# 确定要扫描的目录（目录名 = 向量库名）
|
||
if collection:
|
||
subdirs = [collection]
|
||
else:
|
||
# 列出所有文档目录
|
||
subdirs = []
|
||
if os.path.exists(DOCUMENTS_PATH):
|
||
for d in os.listdir(DOCUMENTS_PATH):
|
||
if os.path.isdir(os.path.join(DOCUMENTS_PATH, d)):
|
||
subdirs.append(d)
|
||
|
||
documents = []
|
||
supported_extensions = {'.pdf', '.docx', '.doc', '.xlsx', '.txt'}
|
||
|
||
for subdir in subdirs:
|
||
level_dir = os.path.join(DOCUMENTS_PATH, subdir)
|
||
if not os.path.exists(level_dir):
|
||
continue
|
||
|
||
# 目录名即向量库名
|
||
coll_name = subdir
|
||
|
||
for filename in os.listdir(level_dir):
|
||
ext = os.path.splitext(filename)[1].lower()
|
||
if ext not in supported_extensions:
|
||
continue
|
||
|
||
filepath = os.path.join(level_dir, filename)
|
||
try:
|
||
stat = os.stat(filepath)
|
||
documents.append({
|
||
"filename": filename,
|
||
"collection": coll_name,
|
||
"path": f"{subdir}/{filename}",
|
||
"size": stat.st_size,
|
||
"last_modified": datetime.fromtimestamp(stat.st_mtime).isoformat()
|
||
})
|
||
except Exception as e:
|
||
logger.warning(f"读取文件信息失败: {filename}, {e}")
|
||
|
||
# 按修改时间倒序
|
||
documents.sort(key=lambda x: x['last_modified'], reverse=True)
|
||
|
||
return success_response(data={"documents": documents, "total": len(documents)})
|
||
|
||
|
||
@document_bp.route('/documents/<path:doc_path>/status', methods=['GET'])
|
||
@require_gateway_auth
|
||
def get_document_status(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
获取文件处理状态
|
||
|
||
查询指定文档的向量化状态和切片数量。
|
||
|
||
Args:
|
||
doc_path: 文档相对路径（collection/filename）
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"status": "processed|pending|error",
|
||
"chunk_count": N,
|
||
"last_processed": "ISO 8601"
|
||
}
|
||
"""
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
# 解析路径
|
||
parts = doc_path.split('/')
|
||
if len(parts) < 2:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
|
||
|
||
subdir = parts[0]
|
||
filename = '/'.join(parts[1:])
|
||
|
||
# 目录名即向量库名
|
||
collection = subdir
|
||
|
||
# 获取文档信息
|
||
from config import DOCUMENTS_PATH
|
||
file_on_disk = os.path.isfile(os.path.join(DOCUMENTS_PATH, subdir, filename))
|
||
|
||
doc_info = kb_manager.get_document_info(collection, filename)
|
||
|
||
if not doc_info:
|
||
if file_on_disk:
|
||
return success_response(data={
|
||
"status": "unprocessed",
|
||
"chunk_count": 0,
|
||
"last_processed": None
|
||
})
|
||
return error_response("NOT_FOUND", NOT_FOUND, "文档不存在", http_status=404)
|
||
|
||
return success_response(data={
|
||
"status": doc_info.get("status", "unknown"),
|
||
"chunk_count": doc_info.get("total_chunks", 0),
|
||
"last_processed": doc_info.get("effective_date") or doc_info.get("version")
|
||
})
|
||
|
||
|
||
@document_bp.route('/documents/<path:doc_path>', methods=['PUT'])
|
||
@require_gateway_auth
|
||
def update_document(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
更新文件（重新上传覆盖）
|
||
|
||
替换现有文件内容，并触发重新向量化。
|
||
|
||
Args:
|
||
doc_path: 文档相对路径（collection/filename）
|
||
|
||
表单参数:
|
||
file: 新文件（必需）
|
||
|
||
Returns:
|
||
{"success": true, "message": "文件已更新"}
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
|
||
if 'file' not in request.files:
|
||
return error_response("NO_FILE", NO_FILE, "没有上传文件", http_status=400)
|
||
|
||
file = request.files['file']
|
||
if file.filename == '':
|
||
return error_response("NO_FILE_SELECTED", NO_FILE_SELECTED, "没有选择文件", http_status=400)
|
||
|
||
# 文件类型校验
|
||
ext = os.path.splitext(file.filename)[1].lower()
|
||
if ext not in ALLOWED_EXTENSIONS:
|
||
return error_response("UNSUPPORTED_FORMAT", UNSUPPORTED_FORMAT, f"不支持的文件类型: {ext}", http_status=400)
|
||
|
||
# 文件大小校验
|
||
file.seek(0, 2) # 跳到文件末尾获取大小
|
||
file_size = file.tell()
|
||
file.seek(0) # 回到文件开头
|
||
if file_size > MAX_FILE_SIZE:
|
||
return error_response("FILE_TOO_LARGE", FILE_TOO_LARGE, f"文件过大（最大 {MAX_FILE_SIZE // 1024 // 1024}MB）", http_status=400)
|
||
|
||
# 解析路径
|
||
parts = doc_path.split('/')
|
||
if len(parts) < 2:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
|
||
|
||
try:
|
||
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
|
||
except ValueError:
|
||
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
|
||
if not os.path.exists(filepath):
|
||
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
|
||
|
||
# 覆盖文件
|
||
file.save(filepath)
|
||
|
||
# 触发重新向量化
|
||
sync_service = _get_sync_service()
|
||
if sync_service:
|
||
try:
|
||
subdir = parts[0]
|
||
filename = '/'.join(parts[1:])
|
||
from knowledge.sync import DocumentChange, ChangeType
|
||
change = DocumentChange(
|
||
document_id=doc_path,
|
||
document_name=filename,
|
||
change_type=ChangeType.MODIFIED,
|
||
old_hash=None,
|
||
new_hash=sync_service.calculate_file_hash(filepath),
|
||
change_time=datetime.now()
|
||
)
|
||
sync_service.process_change(change)
|
||
except Exception as e:
|
||
logger.warning(f"重新向量化失败: {e}")
|
||
|
||
return success_response(message="文件已更新")
|
||
|
||
|
||
@document_bp.route('/documents/<path:doc_path>', methods=['DELETE'])
|
||
@require_gateway_auth
|
||
def delete_document(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
删除文档
|
||
|
||
同时删除向量库中的切片和物理文件。
|
||
|
||
Args:
|
||
doc_path: 文档相对路径（collection/filename）
|
||
|
||
Returns:
|
||
{"success": true, "message": "文档已删除"}
|
||
|
||
Note:
|
||
此操作不可逆，请谨慎使用
|
||
"""
|
||
from config import DOCUMENTS_PATH
|
||
|
||
# 解析路径
|
||
parts = doc_path.split('/')
|
||
if len(parts) < 2:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
|
||
|
||
subdir = parts[0]
|
||
filename = '/'.join(parts[1:])
|
||
|
||
# 目录名即向量库名
|
||
collection = subdir
|
||
|
||
try:
|
||
filepath = _validate_doc_path(doc_path, DOCUMENTS_PATH)
|
||
except ValueError:
|
||
return error_response("FORBIDDEN", FORBIDDEN, "非法路径", http_status=403)
|
||
if not os.path.exists(filepath):
|
||
return error_response("NOT_FOUND", NOT_FOUND, "文件不存在", http_status=404)
|
||
|
||
try:
|
||
# 1. 从向量库删除（source 存的是文件名，不是完整路径）
|
||
kb_manager = _get_kb_manager()
|
||
if kb_manager:
|
||
kb_manager.delete_document(collection, filename)
|
||
|
||
# 2. 删除文件
|
||
os.remove(filepath)
|
||
|
||
return success_response(status_code=DELETE_SUCCESS, message="文档已删除")
|
||
|
||
except Exception as e:
|
||
logger.error(f"删除文档异常: {e}")
|
||
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败，请稍后重试", http_status=500)
|
||
|
||
|
||
@document_bp.route('/documents/<path:doc_path>/chunks', methods=['GET'])
|
||
@require_gateway_auth
|
||
def list_document_chunks(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
查看文件切片
|
||
|
||
返回指定文档的所有切片内容，用于调试和验证。
|
||
|
||
Args:
|
||
doc_path: 文档相对路径（collection/filename）
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"document_id": "...",
|
||
"collection": "...",
|
||
"chunks": [...],
|
||
"total": N
|
||
}
|
||
"""
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
# 解析路径
|
||
parts = doc_path.split('/')
|
||
if len(parts) < 2:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径", http_status=400)
|
||
|
||
subdir = parts[0]
|
||
# 目录名即向量库名
|
||
collection = subdir
|
||
|
||
chunks = kb_manager.get_document_chunks(collection, os.path.basename(doc_path))
|
||
|
||
return success_response(data={
|
||
"document_id": doc_path,
|
||
"collection": collection,
|
||
"chunks": chunks,
|
||
"total": len(chunks)
|
||
})
|
||
|
||
|
||
@document_bp.route('/documents/<path:doc_path>/preview', methods=['GET'])
|
||
@require_gateway_auth
|
||
def preview_document(doc_path: str) -> Tuple[Any, int]:
|
||
"""
|
||
文档预览接口（支持按切片序号跳转）
|
||
|
||
用于前端引用溯源点击跳转：给定 chunk_index，返回目标切片及其上下文。
|
||
复用现有切片查询逻辑，不新增存储。
|
||
|
||
Query Params:
|
||
chunk_index (int): 目标切片序号（来自 citation 的 chunk_index 字段）
|
||
context (int): 上下文切片数，默认 2（前后各取 2 个）
|
||
|
||
Returns:
|
||
{
|
||
"success": true,
|
||
"collection": "...",
|
||
"source": "文件名",
|
||
"total_chunks": N,
|
||
"target_index": M,
|
||
"chunks": [
|
||
{"id": "...", "content": "...", "metadata": {...}, "is_target": true/false},
|
||
...
|
||
]
|
||
}
|
||
"""
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
# 解析路径
|
||
parts = doc_path.split('/')
|
||
if len(parts) < 2:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "无效的文档路径，格式: collection/filename", http_status=400)
|
||
|
||
collection = parts[0]
|
||
filename = os.path.basename(doc_path)
|
||
|
||
# 查询参数
|
||
chunk_index_str = request.args.get('chunk_index')
|
||
try:
|
||
context_count = max(0, min(int(request.args.get('context', 2)), 10))
|
||
except (ValueError, TypeError):
|
||
context_count = 2
|
||
|
||
# 获取所有切片
|
||
all_chunks = kb_manager.get_document_chunks(collection, filename)
|
||
if not all_chunks:
|
||
return error_response("NOT_FOUND", NOT_FOUND, f"文档 '{filename}' 不存在或无切片", http_status=404)
|
||
|
||
total = len(all_chunks)
|
||
|
||
# 如果未指定 chunk_index，返回前 5 个切片作为概览
|
||
if chunk_index_str is None:
|
||
preview_chunks = all_chunks[:5]
|
||
for c in preview_chunks:
|
||
c['is_target'] = False
|
||
return success_response(data={
|
||
"collection": collection,
|
||
"source": filename,
|
||
"total_chunks": total,
|
||
"target_index": None,
|
||
"chunks": preview_chunks
|
||
})
|
||
|
||
# 定位目标切片 —— 按 meta.chunk_index 排序后查找，避免数组下标与 chunk_index 不一致
|
||
try:
|
||
target_chunk_index = int(chunk_index_str)
|
||
except ValueError:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "chunk_index 必须为整数", http_status=400)
|
||
|
||
# 按 chunk_index 排序（Chroma 返回顺序不保证有序）
|
||
all_chunks.sort(key=lambda c: c.get('metadata', {}).get('chunk_index', 0))
|
||
|
||
# 按 meta.chunk_index 查找目标切片在排序后数组中的实际位置
|
||
target_pos = None
|
||
for i, c in enumerate(all_chunks):
|
||
if c.get('metadata', {}).get('chunk_index') == target_chunk_index:
|
||
target_pos = i
|
||
break
|
||
|
||
# 回退：直接用数组下标（兼容旧数据无 chunk_index 字段的情况）
|
||
if target_pos is None:
|
||
if 0 <= target_chunk_index < total:
|
||
target_pos = target_chunk_index
|
||
else:
|
||
max_idx = max(
|
||
(c.get('metadata', {}).get('chunk_index', 0) for c in all_chunks),
|
||
default=total - 1
|
||
)
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, f"chunk_index={target_chunk_index} 未找到对应切片 (可用范围 0-{max_idx})", http_status=400)
|
||
|
||
# 截取上下文窗口
|
||
start = max(0, target_pos - context_count)
|
||
end = min(total, target_pos + context_count + 1)
|
||
window = all_chunks[start:end]
|
||
|
||
# 标记目标切片
|
||
for i, c in enumerate(window):
|
||
c['is_target'] = (start + i == target_pos)
|
||
|
||
return success_response(data={
|
||
"collection": collection,
|
||
"source": filename,
|
||
"total_chunks": total,
|
||
"target_index": target_chunk_index,
|
||
"chunks": window
|
||
})
|
||
|
||
|
||
# ==================== 切片管理 ====================
|
||
|
||
@document_bp.route('/chunks', methods=['POST'])
|
||
@require_gateway_auth
|
||
def create_chunk() -> Tuple[Any, int]:
|
||
"""
|
||
新增切片
|
||
|
||
手动向向量库添加一个切片，用于补充或修正内容。
|
||
|
||
请求体:
|
||
{
|
||
"collection": "向量库名称",
|
||
"content": "切片内容",
|
||
"metadata": {} // 可选
|
||
}
|
||
|
||
Returns:
|
||
{"success": true, "chunk_id": "...", "message": "切片已添加"}
|
||
"""
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
data = request.json or {}
|
||
collection = data.get('collection')
|
||
content = data.get('content')
|
||
metadata = data.get('metadata', {})
|
||
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
|
||
if not content:
|
||
return error_response("NO_CONTENT", NO_CONTENT, "切片内容不能为空", http_status=400)
|
||
|
||
chunk_id = kb_manager.add_chunk(collection, content, metadata)
|
||
|
||
return success_response(data={"chunk_id": chunk_id}, message="切片已添加")
|
||
|
||
|
||
@document_bp.route('/chunks/<chunk_id>', methods=['PUT'])
|
||
@require_gateway_auth
|
||
def update_chunk(chunk_id: str) -> Tuple[Any, int]:
|
||
"""
|
||
修改切片
|
||
|
||
更新指定切片的内容或元数据。
|
||
|
||
Args:
|
||
chunk_id: 切片 ID
|
||
|
||
请求体:
|
||
{
|
||
"collection": "向量库名称",
|
||
"content": "新内容", // 可选
|
||
"metadata": {} // 可选
|
||
}
|
||
|
||
Returns:
|
||
{"success": true, "message": "切片已更新"}
|
||
"""
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
data = request.json or {}
|
||
collection = data.get('collection')
|
||
content = data.get('content')
|
||
metadata = data.get('metadata')
|
||
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
|
||
|
||
success = kb_manager.update_chunk(collection, chunk_id, content=content, metadata=metadata)
|
||
|
||
if success:
|
||
return success_response(message="切片已更新")
|
||
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "更新失败", http_status=500)
|
||
|
||
|
||
@document_bp.route('/chunks/<chunk_id>', methods=['DELETE'])
|
||
@require_gateway_auth
|
||
def delete_chunk(chunk_id: str) -> Tuple[Any, int]:
|
||
"""
|
||
删除切片
|
||
|
||
从向量库中移除指定切片。如果删除后该文件没有其他切片，
|
||
同时清理文档哈希记录，以便同步服务能重新检测该文件。
|
||
|
||
Args:
|
||
chunk_id: 切片 ID
|
||
|
||
请求体或查询参数:
|
||
collection: 向量库名称（必需）
|
||
|
||
Returns:
|
||
{"success": true, "message": "切片已删除"}
|
||
"""
|
||
data = request.get_json(silent=True) or {}
|
||
collection = data.get('collection') if data else None
|
||
|
||
if not collection:
|
||
collection = request.args.get('collection')
|
||
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
|
||
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
# 删除切片，返回 (success, source_file)
|
||
success, source_file = kb_manager.delete_chunk(collection, chunk_id)
|
||
|
||
if success and source_file:
|
||
# 检查该文件是否还有其他切片
|
||
remaining_chunks = kb_manager.list_chunks(collection, limit=100000)
|
||
has_other_chunks = any(
|
||
c.get('metadata', {}).get('source') == source_file
|
||
for c in remaining_chunks
|
||
)
|
||
|
||
# 如果没有其他切片了，删除哈希记录
|
||
if not has_other_chunks:
|
||
try:
|
||
from knowledge.sync import SyncDatabase
|
||
sync_db = SyncDatabase()
|
||
document_id = f"{collection}/{source_file}"
|
||
sync_db.delete_document_hash(document_id)
|
||
logger.info(f"清理文档哈希记录: {document_id}")
|
||
except Exception as e:
|
||
logger.warning(f"清理哈希记录失败: {e}")
|
||
|
||
if success:
|
||
return success_response(status_code=DELETE_SUCCESS, message="切片已删除")
|
||
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "删除失败", http_status=500)
|
||
|
||
|
||
@document_bp.route('/chunks/batch', methods=['DELETE'])
|
||
@require_gateway_auth
|
||
def delete_chunks_by_source() -> Tuple[Any, int]:
|
||
"""
|
||
批量删除指定文件的所有切片
|
||
|
||
删除指定向量库中某个文件的所有切片，并清理哈希记录。
|
||
相比逐个删除，批量删除更高效，避免前端超时。
|
||
|
||
请求体:
|
||
{
|
||
"collection": "向量库名称",
|
||
"source": "文件名"
|
||
}
|
||
|
||
Returns:
|
||
{"success": true, "deleted_count": N, "message": "..."}
|
||
"""
|
||
data = request.get_json(silent=True) or {}
|
||
collection = data.get('collection')
|
||
source = data.get('source')
|
||
|
||
if not collection:
|
||
return error_response("NO_COLLECTION", NO_COLLECTION, "请指定向量库 (collection)", http_status=400)
|
||
if not source:
|
||
return error_response("BAD_REQUEST", BAD_REQUEST, "请指定文件名 (source)", http_status=400)
|
||
|
||
kb_manager = _get_kb_manager()
|
||
if not kb_manager:
|
||
return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "知识库管理器未初始化", http_status=503)
|
||
|
||
# 批量删除该文件的所有切片
|
||
try:
|
||
deleted_count = kb_manager.delete_chunks_by_source(collection, source)
|
||
|
||
# 清理哈希记录
|
||
if deleted_count > 0:
|
||
try:
|
||
from knowledge.sync import SyncDatabase
|
||
sync_db = SyncDatabase()
|
||
# 统一使用正斜杠格式（与 sync.py scan_documents 一致）
|
||
document_id = f"{collection}/{source}"
|
||
sync_db.delete_document_hash(document_id)
|
||
logger.info(f"清理文档哈希记录: {document_id}")
|
||
except Exception as e:
|
||
logger.warning(f"清理哈希记录失败: {e}")
|
||
|
||
return success_response(data={"deleted_count": deleted_count}, message=f"已删除 {deleted_count} 个切片")
|
||
except Exception as e:
|
||
logger.error(f"操作异常: {e}")
|
||
return error_response("INTERNAL_ERROR", INTERNAL_ERROR, "操作失败，请稍后重试", http_status=500)
|