""" 多向量库管理 API 本模块提供向量库的 CRUD 操作和文档同步功能,支持: - 创建、查询、修改、删除向量库 - 文档向量化同步 - 文档版本管理(废止/恢复) - 知识库路由测试 路由列表: GET /collections : 获取向量库列表 POST /collections : 创建向量库 PUT /collections/ : 修改向量库信息 DELETE /collections/ : 删除向量库 GET /collections//documents : 获取向量库文档列表 GET /collections//chunks : 获取向量库切片列表 POST /documents/sync : 触发文档同步 POST /kb/route : 测试知识库路由(调试) 文档版本管理: POST /collections//documents//deprecate : 废止文档 POST /collections//documents//restore : 恢复文档 GET /collections//documents//versions : 版本历史 架构说明: - 权限验证由后端网关完成,RAG 服务不做权限判断 - 每个部门可拥有独立的向量库(多租户架构) - 向量库元数据存储在 kb_metadata.json Example: curl -X GET http://localhost:5001/collections \\ -H "Authorization: Bearer mock-token-admin" """ import os from typing import Tuple, Optional, Any from flask import Blueprint, request, jsonify, current_app import logging from core.status_codes import ( SUCCESS, CREATED, DELETE_SUCCESS, UPDATE_SUCCESS, SYNC_SUCCESS, BAD_REQUEST, NOT_FOUND, COLLECTION_NOT_FOUND, NO_COLLECTION, TASK_CONFLICT, INTERNAL_ERROR, SERVICE_UNAVAILABLE, SYNC_ERROR, REINDEX_ERROR ) from api.response_utils import success_response, error_response logger = logging.getLogger(__name__) from auth.gateway import require_gateway_auth kb_bp = Blueprint('kb', __name__) # 延迟初始化缓存 _kb_manager = None _kb_router = None _kb_checked = False _has_multi_kb = False def _check_multi_kb() -> bool: """ 检查多向量库模块是否可用 延迟导入知识库管理器和路由器,避免循环依赖。 结果缓存在模块级变量中,后续调用直接返回缓存。 Returns: 多向量库可用返回 True,否则返回 False """ global _kb_checked, _has_multi_kb, _kb_manager, _kb_router if not _kb_checked: try: from knowledge.manager import get_kb_manager from knowledge.router import get_kb_router _kb_manager = get_kb_manager() _kb_router = get_kb_router() _has_multi_kb = True except ImportError as e: logger.warning(f"多向量库模块导入失败: {e}") _has_multi_kb = False _kb_checked = True return _has_multi_kb def _require_multi_kb() -> Tuple[Optional[Any], Optional[Any], Optional[Tuple]]: """ 检查多向量库是否可用,返回管理器和路由器 Returns: 元组 (kb_manager, kb_router, error_response): - 成功时 error_response 为 None - 失败时 kb_manager 和 kb_router 为 None Example: >>> kb_manager, kb_router, err = _require_multi_kb() >>> if err: ... return err """ if not _check_multi_kb(): return None, None, (jsonify({"error": "多向量库模块未启用"}), 503) return _kb_manager, _kb_router, None @kb_bp.route('/collections', methods=['GET']) @require_gateway_auth def list_collections() -> Tuple[Any, int]: """ 获取向量库列表 返回当前用户可访问的所有向量库信息,包括: - 向量库名称和显示名称 - 文档数量 - 所属部门 - 创建时间 - 描述信息 Returns: JSON 响应,包含 collections 列表和 total 总数 Example: GET /collections Response: {"collections": [...], "total": 3} """ kb_manager, _, err = _require_multi_kb() if err: return err user = request.current_user # 获取所有向量库(权限由后端网关管理) all_collections = kb_manager.list_collections() result = [] for coll in all_collections: result.append({ "name": coll.name, "display_name": coll.display_name, "document_count": coll.document_count, "department": coll.department, "created_at": coll.created_at, "description": coll.description }) return success_response(data={"collections": result, "total": len(result)}) @kb_bp.route('/collections', methods=['POST']) @require_gateway_auth def create_collection() -> Tuple[Any, int]: """ 创建新向量库 请求体: { "name": "向量库标识(英文)", "display_name": "显示名称", "department": "所属部门", "description": "描述信息" } 注意: - 名称只能包含字母、数字、下划线和连字符 - 名称不能为空 Returns: 成功: {"success": true, "message": "...", "name": "..."} (201) 失败: {"error": "..."} (400) """ kb_manager, _, err = _require_multi_kb() if err: return err data = request.json or {} name = data.get('name', '').strip() display_name = data.get('display_name', '') department = data.get('department', '') description = data.get('description', '') if not name: return error_response("INVALID_NAME", BAD_REQUEST, "向量库名称不能为空", http_status=400) # 验证名称格式(ChromaDB 限制) if not name.replace('_', '').replace('-', '').isalnum(): return error_response("INVALID_NAME_FORMAT", BAD_REQUEST, "向量库名称只能包含字母、数字、下划线和连字符", http_status=400) success, message = kb_manager.create_collection( name, display_name, department, description ) if success: return success_response(data={"name": name}, status_code=CREATED, message=message, http_status=201) return error_response("CREATE_FAILED", BAD_REQUEST, message, http_status=400) @kb_bp.route('/collections/', methods=['PUT']) @require_gateway_auth def update_collection(kb_name: str) -> Tuple[Any, int]: """ 修改向量库信息 Args: kb_name: 向量库名称 请求体: { "display_name": "新显示名称", "description": "新描述" } Returns: 成功: {"success": true, "message": "向量库信息已更新"} 失败: {"error": "..."} (404/500) """ kb_manager, _, err = _require_multi_kb() if err: return err data = request.json or {} display_name = data.get('display_name') description = data.get('description') # 检查向量库是否存在 collections = kb_manager.list_collections() if not any(c.name == kb_name for c in collections): return error_response("COLLECTION_NOT_FOUND", COLLECTION_NOT_FOUND, f"向量库 '{kb_name}' 不存在", http_status=404) # 更新元数据 success = kb_manager.update_collection_metadata( kb_name, display_name=display_name, description=description ) if success: return success_response(data=None, status_code=UPDATE_SUCCESS, message="向量库信息已更新") return error_response("UPDATE_FAILED", INTERNAL_ERROR, "更新失败", http_status=500) @kb_bp.route('/collections/', methods=['DELETE']) @require_gateway_auth def delete_collection(kb_name: str) -> Tuple[Any, int]: """ 删除向量库 Args: kb_name: 向量库名称 查询参数: delete_documents: 是否删除文档源文件(默认 false) Returns: 成功: {"success": true, "message": "...", "deleted_documents": bool} 失败: {"error": "..."} (400) """ kb_manager, _, err = _require_multi_kb() if err: return err # 获取参数 delete_documents = request.args.get('delete_documents', 'false').lower() == 'true' success, message = kb_manager.delete_collection(kb_name, delete_documents) if success: return success_response(data={"deleted_documents": delete_documents}, status_code=DELETE_SUCCESS, message=message) return error_response("DELETE_FAILED", BAD_REQUEST, message, http_status=400) @kb_bp.route('/collections//documents', methods=['GET']) @require_gateway_auth def list_collection_documents(kb_name: str) -> Tuple[Any, int]: """ 获取向量库中的文档列表 Args: kb_name: 向量库名称 Returns: {"collection": "...", "documents": [...], "total": N} """ kb_manager, _, err = _require_multi_kb() if err: return err documents = kb_manager.list_documents(kb_name) return success_response(data={"collection": kb_name, "documents": documents, "total": len(documents)}) @kb_bp.route('/collections//chunks', methods=['GET']) @require_gateway_auth def list_collection_chunks(kb_name: str) -> Tuple[Any, int]: """ 获取向量库中的切片列表 Args: kb_name: 向量库名称 查询参数: document_id: 过滤指定文档的切片(可选) limit: 返回数量限制(默认 100) offset: 偏移量(默认 0) Returns: {"collection": "...", "chunks": [...], "total": N} """ kb_manager, _, err = _require_multi_kb() if err: return err # 可选过滤参数 document_id = request.args.get('document_id') limit = request.args.get('limit', 100, type=int) offset = request.args.get('offset', 0, type=int) chunks = kb_manager.list_chunks(kb_name, document_id=document_id, limit=limit, offset=offset) return success_response(data={"collection": kb_name, "chunks": chunks, "total": len(chunks)}) @kb_bp.route('/documents/sync', methods=['POST']) @require_gateway_auth def sync_documents() -> Tuple[Any, int]: """ 触发文档向量化同步(异步任务) 立即返回 task_id,后台线程执行同步。 请求体: { "collection": "向量库名称" // 可选,不传则同步所有 } Returns: {"success": true, "data": {"task_id": "xxx", "message": "..."}} """ kb_manager, _, err = _require_multi_kb() if err: return err from config import DOCUMENTS_PATH data = request.json or {} target_collection = data.get('collection') # 确定要同步的向量库 if target_collection: collections_to_sync = [target_collection] else: all_collections = kb_manager.list_collections() collections_to_sync = [c.name for c in all_collections] if not collections_to_sync: return error_response("NO_COLLECTION", NO_COLLECTION, "没有可同步的向量库", http_status=400) sync_service = current_app.config.get('SYNC_SERVICE') if not sync_service: return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503, results=[{"collection": c, "status": "warning", "message": "同步服务不可用"} for c in collections_to_sync], synced_count=0 ) from core.task_registry import get_registry registry = get_registry() # 检查是否有正在运行的同步任务 running = registry.list_tasks(status='running', task_type='sync', limit=1) if running: return error_response("TASK_RUNNING", TASK_CONFLICT, f"同步任务正在执行中 (task_id: {running[0].id})", http_status=409) desc = f"文档同步: {target_collection or '所有向量库'}" task = registry.create_task('sync', desc) def _do_sync(task, sync_svc): processed = [0] def on_change(change): processed[0] += 1 registry.update_progress( task.id, current=processed[0], stage='处理文件', message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}" ) old_callback = sync_svc.on_change_callback sync_svc.on_change_callback = on_change try: registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...') sync_result = sync_svc.sync_now() return { 'collection': target_collection or 'all', 'status': 'success', 'message': f"同步完成: 处理 {sync_result.documents_processed} 个文档", 'details': { 'added': sync_result.documents_added, 'modified': sync_result.documents_modified, 'deleted': sync_result.documents_deleted, 'errors': sync_result.errors, } } finally: sync_svc.on_change_callback = old_callback registry.start_task(task.id, _do_sync, sync_service) return success_response( data={"task_id": task.id, "message": f"同步任务已启动,通过 GET /tasks/{task.id} 查询进度"}, status_code=SYNC_SUCCESS, message="同步任务已启动" ) @kb_bp.route('/debug/scan', methods=['GET']) @require_gateway_auth def debug_scan() -> Tuple[Any, int]: """ 调试:查看同步服务扫描的文件 返回文档目录的文件列表和同步服务状态, 用于排查文档同步问题。 Returns: { "documents_path": "...", "exists": bool, "files": [...], "sync_service_path": "...", "scanned_count": N } """ from config import DOCUMENTS_PATH import os result = { "documents_path": DOCUMENTS_PATH, "exists": os.path.exists(DOCUMENTS_PATH), "files": [] } if os.path.exists(DOCUMENTS_PATH): for root, dirs, files in os.walk(DOCUMENTS_PATH): for f in files: fp = os.path.join(root, f) rel = os.path.relpath(fp, DOCUMENTS_PATH) result["files"].append({ "rel_path": rel, "size": os.path.getsize(fp), "ext": os.path.splitext(f)[1].lower() }) # 也检查同步服务的路径 sync_service = current_app.config.get('SYNC_SERVICE') if sync_service: result["sync_service_path"] = sync_service.documents_path result["sync_service_path_exists"] = os.path.exists(sync_service.documents_path) # 直接调用 scan_documents try: scanned = sync_service.scan_documents() result["scanned_count"] = len(scanned) result["scanned_ids"] = list(scanned.keys())[:10] except Exception as e: logger.error(f"扫描异常: {e}") result["scan_error"] = "扫描失败" return jsonify(result) @kb_bp.route('/collections//reindex', methods=['POST']) @require_gateway_auth def reindex_collection(kb_name: str) -> Tuple[Any, int]: """ 强制重新向量化指定集合的所有文档(异步任务) 清除该集合的文档哈希记录,触发完整重新索引。 立即返回 task_id,后台线程执行重建。 Args: kb_name: 向量库名称 Returns: {"success": true, "data": {"task_id": "xxx", "message": "..."}} """ kb_manager, _, err = _require_multi_kb() if err: return err from config import DOCUMENTS_PATH # 清除该集合的文档哈希记录(同步完成,很快) try: from data.db import get_connection with get_connection("knowledge") as conn: cursor = conn.cursor() escaped_kb = kb_name.replace('%', '\\%').replace('_', '\\_') cursor.execute("DELETE FROM document_hashes WHERE document_id LIKE ? ESCAPE '\\' OR document_id LIKE ? ESCAPE '\\'", (f"{escaped_kb}/%", f"{escaped_kb}\\%")) deleted = cursor.rowcount logger.info(f"已清除 {deleted} 条哈希记录: {kb_name}") except Exception as e: logger.warning(f"清除哈希记录失败: {e}") sync_service = current_app.config.get('SYNC_SERVICE') if not sync_service: return error_response("SERVICE_UNAVAILABLE", SERVICE_UNAVAILABLE, "同步服务不可用", http_status=503) from core.task_registry import get_registry registry = get_registry() # 检查是否有正在运行的重建任务 running = registry.list_tasks(status='running', task_type='reindex', limit=1) if running: return error_response("TASK_RUNNING", TASK_CONFLICT, f"重建任务正在执行中 (task_id: {running[0].id})", http_status=409) task = registry.create_task('reindex', f'重建索引: {kb_name}') def _do_reindex(task, sync_svc, kb): """后台执行重建索引""" processed = [0] def on_change(change): processed[0] += 1 registry.update_progress( task.id, current=processed[0], stage='重新索引', message=f"已处理: {change.document_name if hasattr(change, 'document_name') else change.document_id}" ) old_callback = sync_svc.on_change_callback sync_svc.on_change_callback = on_change try: registry.update_progress(task.id, stage='扫描文档', message='正在检测变更...') result = sync_svc.sync_now() return { 'message': f"重新索引完成: 处理 {result.documents_processed} 个文档", 'documents_processed': result.documents_processed, 'documents_added': result.documents_added, 'errors': result.errors, } finally: sync_svc.on_change_callback = old_callback registry.start_task(task.id, _do_reindex, sync_service, kb_name) return success_response( data={"task_id": task.id, "message": f"重建索引任务已启动: {kb_name},通过 GET /tasks/{task.id} 查询进度"}, status_code=SYNC_SUCCESS, message="重建索引任务已启动" ) @kb_bp.route('/kb/route', methods=['POST']) @require_gateway_auth def test_routing() -> Tuple[Any, int]: """ 测试知识库路由(调试用) 分析查询意图,返回目标向量库列表和意图分析结果。 请求体: {"query": "用户查询"} Returns: { "query": "...", "user_role": "...", "user_department": "...", "target_collections": [...], "intent": { "is_general": bool, "department": "...", "confidence": float, "keywords": [...], "reason": "..." } } """ _, kb_router, err = _require_multi_kb() if err: return err from knowledge.router import route_query user = request.current_user data = request.json or {} query = data.get('query', '') if not query: return error_response("MISSING_PARAMS", BAD_REQUEST, "请提供查询内容", http_status=400) # 获取路由结果 target_kbs = route_query( query, user.get("role", "user"), user.get("department", "") ) # 获取意图分析 intent = kb_router.analyze_intent(query) return success_response(data={ "query": query, "user_role": user.get("role"), "user_department": user.get("department", ""), "target_collections": target_kbs, "intent": { "is_general": intent.is_general, "department": intent.department, "confidence": intent.confidence, "keywords": intent.keywords, "reason": intent.reason } }) # ==================== 文档版本管理 API ==================== @kb_bp.route('/collections//documents//deprecate', methods=['POST']) @require_gateway_auth def deprecate_document(kb_name: str, filename: str) -> Tuple[Any, int]: """ 废止文档(软删除) 将文档标记为已废止,相关切片在检索时被过滤。 文档数据保留,可通过 restore_document 恢复。 Args: kb_name: 向量库名称 filename: 文档文件名 请求体: {"reason": "废止原因"} Returns: { "success": true, "deprecated_chunks": N, "document_id": "...", "collection": "...", "deprecated_date": "ISO 8601 时间" } """ kb_manager, _, err = _require_multi_kb() if err: return err user = request.current_user data = request.json or {} reason = data.get('reason', '文档已废止') try: result = kb_manager.deprecate_document( kb_name, filename, reason, deprecated_by=user.get('user_id', 'unknown') ) return success_response(data=result) except Exception as e: logger.error(f"操作异常: {e}") return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500) @kb_bp.route('/collections//documents//restore', methods=['POST']) @require_gateway_auth def restore_document(kb_name: str, filename: str) -> Tuple[Any, int]: """ 恢复已废止的文档 将已废止的文档恢复为有效状态,相关切片重新参与检索。 Args: kb_name: 向量库名称 filename: 文档文件名 Returns: { "success": true, "restored_chunks": N, "document_id": "...", "collection": "..." } """ kb_manager, _, err = _require_multi_kb() if err: return err try: result = kb_manager.restore_document(kb_name, filename) return success_response(data=result) except Exception as e: logger.error(f"操作异常: {e}") return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500) @kb_bp.route('/collections//documents//versions', methods=['GET']) @require_gateway_auth def get_document_versions(kb_name: str, filename: str) -> Tuple[Any, int]: """ 获取文档版本历史 返回文档的所有版本记录,包括状态变更、时间戳等信息。 Args: kb_name: 向量库名称 filename: 文档文件名 查询参数: limit: 返回数量限制(默认 10) Returns: { "success": true, "document_id": "...", "collection": "...", "versions": [ { "version": "v2", "status": "active", "created_at": "ISO 8601", "chunk_count": N }, ... ], "total": N } """ _, _, err = _require_multi_kb() if err: return err limit = request.args.get('limit', 10, type=int) try: from knowledge.document_versions import get_version_query version_query = get_version_query() versions = version_query.get_document_history(kb_name, filename, limit) versions_data = [v.to_dict() for v in versions] return success_response(data={ "document_id": filename, "collection": kb_name, "versions": versions_data, "total": len(versions_data) }) except Exception as e: logger.error(f"操作异常: {e}") return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500) @kb_bp.route('/collections//update-image-descriptions', methods=['POST']) @require_gateway_auth def update_image_descriptions(kb_name: str) -> Tuple[Any, int]: """ 更新图片切片的轻量级描述 重新提取已入库文档中图片切片的图号/表号信息, 生成新的轻量级描述,提高检索准确度。 适用场景: - 文档入库时未提取图号 - 图号提取规则更新后需要重新处理 Args: kb_name: 向量库名称 Returns: { "success": true, "image_count": N, "updated_count": M } """ kb_manager, _, err = _require_multi_kb() if err: return err try: result = kb_manager.update_image_descriptions(kb_name) return success_response(data=result) except Exception as e: logger.error(f"操作异常: {e}") return error_response("OPERATION_FAILED", INTERNAL_ERROR, "操作失败,请稍后重试", http_status=500) @kb_bp.route('/collections/sync-vlm-cache', methods=['POST']) @require_gateway_auth def sync_vlm_cache() -> Tuple[Any, int]: """ 同步 VLM 缓存到向量库元数据 将 .data/cache/vlm/ 中已生成的图片描述同步到向量库的 vlm_desc 字段。 同步后,检索时可直接从向量库获取 VLM 描述,无需读取文件系统。 请求参数: - collection: 可选,指定同步的集合名,不传则同步所有集合 - dry_run: 可选,设为 true 只预览不执行 Returns: { "success": true, "total_cache_files": N, "synced_count": M, "skipped_count": K, "details": [...] } """ from pathlib import Path import hashlib import json as json_module kb_manager, _, err = _require_multi_kb() if err: return err data = request.get_json() or {} collection_filter = data.get('collection') dry_run = data.get('dry_run', False) vlm_cache_dir = Path(".data/cache/vlm") images_dir = Path(".data/images") if not vlm_cache_dir.exists(): return error_response("VLM_CACHE_NOT_FOUND", BAD_REQUEST, "VLM 缓存目录不存在", http_status=400) # 获取所有 VLM 缓存文件 cache_files = list(vlm_cache_dir.glob("*.txt")) if not cache_files: return success_response(data={"total_cache_files": 0, "synced_count": 0}, message="无 VLM 缓存文件") # 构建图片 MD5 → 文件名 的映射 image_hash_map = {} if images_dir.exists(): for img_file in images_dir.glob("*.png"): img_hash = hashlib.md5(img_file.read_bytes()).hexdigest() image_hash_map[img_hash] = img_file.name # 结果统计 synced_count = 0 skipped_count = 0 details = [] # 获取所有集合 collections = kb_manager.list_collections() if collection_filter: collections = [c for c in collections if c.name == collection_filter] for cache_file in cache_files: cache_hash = cache_file.stem # 文件名即 MD5 # 查找对应的图片文件名 image_filename = image_hash_map.get(cache_hash) if not image_filename: skipped_count += 1 details.append({"cache": cache_file.name, "status": "skipped", "reason": "图片文件不存在"}) continue # 读取 VLM 描述 vlm_desc = cache_file.read_text(encoding='utf-8') # 在所有集合中查找该图片切片 found = False for coll in collections: collection = kb_manager.get_collection(coll.name) if not collection: continue # 查找 image_path 匹配的切片 result = collection.get(where={"image_path": image_filename}) if result['ids']: found = True if not dry_run: # 更新元数据 updated_metadatas = [] for meta in result['metadatas']: meta['vlm_desc'] = vlm_desc meta['has_vlm_desc'] = True updated_metadatas.append(meta) collection.update( ids=result['ids'], metadatas=updated_metadatas ) synced_count += len(result['ids']) details.append({ "cache": cache_file.name, "image": image_filename, "collection": coll.name, "count": len(result['ids']), "status": "synced" if not dry_run else "preview" }) if not found: skipped_count += 1 details.append({"cache": cache_file.name, "image": image_filename, "status": "skipped", "reason": "向量库中未找到对应切片"}) return success_response(data={ "total_cache_files": len(cache_files), "synced_count": synced_count, "skipped_count": skipped_count, "dry_run": dry_run, "details": details[:20] # 只返回前 20 条详情 })