feat(rag): 子章节级图片过滤 + VLM 后台增强 + 意图分析优化

- 图片选择新增 section_path 字段，支持子章节级过滤
- _filter_images_by_answer 增加 primary_sections 参数和叶子节点匹配
- 检索发散检测：primary_leaf_names > 3 时全局阈值+1
- _FIGURE_ANSWER_KEYWORDS 移除单字"图""表"，正则图号兜底防误触发
- lazy_enhance 后台增强流程优化
- 意图分析与 LLM 工具层改进

评测：图片选择 F1 从 52.4% 提升至 66.3%（+13.9pp），Precision +16.5pp

knowledge/collection.py

@@ -331,6 +331,21 @@ class CollectionMixin:
             except Exception as e:
                 logger.warning(f"清理版本记录失败: {e}")
 
+            # 清理不再被引用的图片和 VLM 缓存文件
+            # 注意：此时 ChromaDB collection 已删除，cleanup_image_orphans 会扫描
+            # 所有剩余 collection，仅该 collection 引用的图片会被识别为孤儿
+            try:
+                from knowledge.image_cleanup import cleanup_image_orphans
+                cleanup_result = cleanup_image_orphans(self)
+                if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
+                    logger.info(
+                        f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
+                        f"{cleanup_result['deleted_caches']} VLM缓存, "
+                        f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
+                    )
+            except Exception as e:
+                logger.warning(f"清理孤儿文件失败: {e}")
+
             if kb_name in self._metadata.get("collections", {}):
                 del self._metadata["collections"][kb_name]
             self._save_metadata()

knowledge/document.py

@@ -72,6 +72,18 @@ class DocumentMixin:
         except Exception as e:
             logger.warning(f"清理版本记录失败: {e}")
 
+        # 清理不再被引用的图片和 VLM 缓存文件
+        try:
+            from knowledge.image_cleanup import cleanup_image_orphans
+            cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
+            if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
+                logger.info(
+                    f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
+                    f"{cleanup_result['deleted_caches']} VLM缓存"
+                )
+        except Exception as e:
+            logger.warning(f"清理孤儿文件失败: {e}")
+
         logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
         return deleted
 

knowledge/lazy_enhance.py

@@ -25,7 +25,20 @@ def compute_file_hash(file_path: str) -> str:
         return hashlib.md5(file_path.encode()).hexdigest()
 
 
-async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None) -> str:
+def _get_embedding_model():
+    """从 RAGEngine 获取 embedding 模型（KnowledgeBaseManager 上没有此属性）"""
+    try:
+        from core.engine import get_engine
+        engine = get_engine()
+        if not engine._initialized:
+            engine.initialize()
+        return engine.embedding_model
+    except Exception as e:
+        logger.warning(f"获取 embedding 模型失败: {e}")
+        return None
+
+
+async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None, defer_chromadb: bool = False) -> str:
     """
     懒加载 VLM 描述
 
@@ -36,6 +49,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
         image_path: 图片路径（相对路径或绝对路径）
         kb_name: 知识库名称
         metadata: 图片元数据（包含 section、page、caption、上下文等）
+        defer_chromadb: 为 True 时跳过 ChromaDB 更新（仅写文件缓存），避免后台线程写锁竞争
 
     Returns:
         VLM 生成的图片描述
@@ -49,23 +63,45 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
     else:
         full_image_path = image_path
 
-    # 1. 检查缓存
+    # 1. 检查缓存（空缓存视为无效，需重新生成）
     img_hash = compute_file_hash(full_image_path)
     cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
     if cache_file.exists():
-        logger.info(f"VLM 缓存命中: {image_path}")
-        return cache_file.read_text(encoding='utf-8')
+        cached = cache_file.read_text(encoding='utf-8')
+        if len(cached.strip()) >= 5:
+            logger.info(f"VLM 缓存命中: {image_path}")
+            return cached
+        else:
+            logger.warning(f"VLM 缓存内容过短({len(cached.strip())}字符)，删除并重新生成: {image_path}")
+            try:
+                cache_file.unlink()
+            except OSError:
+                pass
 
     # 2. 调用 VLM（传入元数据）
     logger.info(f"VLM 懒加载: {image_path}")
     kb_manager = get_kb_manager()
     description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
 
-    # 3. 写入缓存
+    # 3. 空描述保护：VLM 返回内容过短时不写入缓存和向量库
+    if not description or len(description.strip()) < 5:
+        logger.warning(f"VLM 返回描述过短({len(description.strip()) if description else 0}字符)，跳过缓存和向量库更新: {image_path}")
+        return description or ''
+
+    # 4. 写入缓存
     VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
     cache_file.write_text(description, encoding='utf-8')
 
-    # 4. 更新向量库（metadata + embedding）
+    # 5. 更新向量库（metadata + embedding），需校验 chunk_id 非空
+    # defer_chromadb=True 时跳过（后台线程只写缓存，避免 SQLite 写锁竞争）
+    if defer_chromadb:
+        logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
+        return description
+
+    if not chunk_id:
+        logger.warning("chunk_id 为空，跳过向量库更新")
+        return description
+
     try:
         collection = kb_manager.get_collection(kb_name)
         result = collection.get(ids=[chunk_id], include=['metadatas'])
@@ -79,7 +115,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
 
             # 更新 embedding（使用 VLM 描述重新计算向量）
             # 这样 VLM 描述中的关键词（如"发电量"）才能参与相似度检索
-            embedding_model = kb_manager.embedding_model
+            embedding_model = _get_embedding_model()
             if embedding_model:
                 new_vector = embedding_model.encode(description).tolist()
                 if isinstance(new_vector[0], list):
@@ -91,20 +127,21 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
                     embeddings=[new_vector],
                     documents=[description]  # 同时更新 document 字段
                 )
-                logger.info(f"已更新向量库 embedding: {chunk_id}")
+                logger.info(f"已更新向量库(embedding+metadata): {chunk_id}")
             else:
                 # 无 embedding 模型时只更新 metadata
                 collection.update(
                     ids=[chunk_id],
                     metadatas=[new_metadata]
                 )
+                logger.info(f"已更新向量库(仅metadata,无embedding模型): {chunk_id}")
     except Exception as e:
         logger.warning(f"更新向量库失败: {e}")
 
     return description
 
 
-async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
+async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str, defer_chromadb: bool = False) -> str:
     """
     懒加载表格摘要
 
@@ -114,50 +151,76 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
         chunk_id: 切片 ID
         table_md: 表格 Markdown 内容
         kb_name: 知识库名称
+        defer_chromadb: 为 True 时跳过 ChromaDB 更新（仅写文件缓存），避免后台线程写锁竞争
 
     Returns:
         LLM 生成的表格摘要
     """
     from knowledge.manager import get_kb_manager
 
-    # 1. 检查缓存
+    # 1. 检查缓存（空缓存视为无效）
     table_hash = hashlib.md5(table_md.encode()).hexdigest()
     cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
     if cache_file.exists():
-        logger.info(f"LLM 缓存命中: {chunk_id}")
-        return cache_file.read_text(encoding='utf-8')
+        cached = cache_file.read_text(encoding='utf-8')
+        if len(cached.strip()) >= 5:
+            logger.info(f"LLM 缓存命中: {chunk_id}")
+            return cached
+        else:
+            logger.warning(f"LLM 缓存内容过短({len(cached.strip())}字符)，删除并重新生成: {chunk_id}")
+            try:
+                cache_file.unlink()
+            except OSError:
+                pass
 
     # 2. 调用 LLM
     logger.info(f"LLM 懒加载: {chunk_id}")
     kb_manager = get_kb_manager()
     summary = kb_manager._generate_table_summary(table_md, None)
 
+    # 空摘要保护
+    if not summary or len(summary.strip()) < 5:
+        logger.warning(f"LLM 返回摘要过短，跳过缓存和向量库更新: {chunk_id}")
+        return summary or ''
+
     # 3. 写入缓存
     LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
     cache_file.write_text(summary, encoding='utf-8')
 
-    # 4. 更新向量库（可选）
+    # 4. 更新向量库，需校验 chunk_id 非空
+    # defer_chromadb=True 时跳过（后台线程只写缓存，避免 SQLite 写锁竞争）
+    if defer_chromadb:
+        logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
+        return summary
+
+    if not chunk_id:
+        logger.warning("chunk_id 为空，跳过表格向量库更新")
+        return summary
     try:
         collection = kb_manager.get_collection(kb_name)
         result = collection.get(ids=[chunk_id], include=['metadatas'])
         if result['metadatas']:
-            # 新增摘要切片
-            embedding_model = kb_manager.embedding_model
-            vector = embedding_model.encode(summary).tolist()
-            if isinstance(vector[0], list):
-                vector = vector[0]
+            # 新增摘要切片（需要 embedding 模型）
+            embedding_model = _get_embedding_model()
+            if embedding_model:
+                vector = embedding_model.encode(summary).tolist()
+                if isinstance(vector[0], list):
+                    vector = vector[0]
 
-            collection.add(
-                ids=[f"{chunk_id}_summary"],
-                embeddings=[vector],
-                documents=[summary],
-                metadatas=[{
-                    **result['metadatas'][0],
-                    'is_summary': True,
-                    'original_doc_id': chunk_id
-                }]
-            )
-            # 更新原切片标记
+                collection.add(
+                    ids=[f"{chunk_id}_summary"],
+                    embeddings=[vector],
+                    documents=[summary],
+                    metadatas=[{
+                        **result['metadatas'][0],
+                        'is_summary': True,
+                        'original_doc_id': chunk_id
+                    }]
+                )
+                logger.info(f"已新增摘要切片(embedding): {chunk_id}_summary")
+            else:
+                logger.info(f"跳过摘要切片(无embedding模型): {chunk_id}")
+            # 更新原切片标记（不依赖 embedding 模型）
             collection.update(
                 ids=[chunk_id],
                 metadatas=[{**result['metadatas'][0], 'has_summary': True}]
@@ -168,7 +231,7 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
     return summary
 
 
-async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
+async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str, defer_chromadb: bool = False):
     """
     检索后增强：按需调用 LLM/VLM
 
@@ -176,23 +239,24 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
         contexts: 检索上下文列表
         query: 用户查询
         kb_name: 知识库名称
+        defer_chromadb: 为 True 时后台线程只写文件缓存，不更新 ChromaDB（避免写锁竞争）
     """
-    for ctx in contexts:
-        meta = ctx.get('meta', {})
-        chunk_type = meta.get('chunk_type', 'text')
-        image_path = meta.get('image_path', '')
+    import re
 
-        # 图片切片：懒加载 VLM 描述
-        if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
-            if image_path:
-                try:
+    for ctx in contexts:
+        try:
+            meta = ctx.get('meta', {})
+            chunk_type = meta.get('chunk_type', 'text')
+            image_path = meta.get('image_path', '')
+
+            # 图片切片：懒加载 VLM 描述
+            if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
+                if image_path:
                     # 从 doc 字段中提取图号（上下文可能包含"见图2.5"等）
                     doc_text = ctx.get('doc', '')
-                    import re
 
                     # 提取图号（从前文/后文中）
                     figure_number = ""
-                    # 匹配 "见图2.5"、"图2.5"、"见图 2.5" 等
                     fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
                     if fig_match:
                         figure_number = fig_match.group(1)
@@ -210,78 +274,73 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
                         'page': meta.get('page'),
                         'caption': meta.get('caption', ''),
                         'source': meta.get('source', ''),
-                        'figure_number': figure_number,  # 添加提取的图号
-                        'doc_text': doc_text  # 添加完整文档文本
+                        'figure_number': figure_number,
+                        'doc_text': doc_text
                     }
                     vlm_desc = await lazy_vlm_description(
-                        meta.get('id', ''),
+                        meta.get('chunk_id', ''),
                         image_path,
                         kb_name,
-                        metadata=image_metadata
+                        metadata=image_metadata,
+                        defer_chromadb=defer_chromadb
                     )
-                    ctx['doc'] = vlm_desc
-                    ctx['vlm_enhanced'] = True
-                except Exception as e:
-                    logger.warning(f"VLM 懒加载失败: {e}")
+                    if vlm_desc:
+                        ctx['doc'] = vlm_desc
+                        ctx['vlm_enhanced'] = True
 
-        # 表格切片：同时处理摘要和关联图片的 VLM 描述
-        elif chunk_type == 'table':
-            doc_text = ctx.get('doc', '')
+            # 表格切片：同时处理摘要和关联图片的 VLM 描述
+            elif chunk_type == 'table':
+                doc_text = ctx.get('doc', '')
 
-            # 1. 懒加载表格摘要（高分切片）
-            if not meta.get('has_summary'):
-                score = meta.get('score', 0)
-                if score > 0.7:  # 只对高相关表格生成摘要
-                    try:
+                # 1. 懒加载表格摘要（高分切片）
+                if not meta.get('has_summary'):
+                    score = ctx.get('score', 0)
+                    if score > 0.7:
                         summary = await lazy_table_summary(
-                            meta.get('id', ''),
+                            meta.get('chunk_id', ''),
                             doc_text,
-                            kb_name
+                            kb_name,
+                            defer_chromadb=defer_chromadb
                         )
-                        # 摘要作为补充信息
-                        ctx['summary'] = summary
-                        ctx['llm_enhanced'] = True
-                    except Exception as e:
-                        logger.warning(f"表格摘要懒加载失败: {e}")
-
-            # 2. 表格有关联图片时，懒加载 VLM 描述
-            if image_path and not meta.get('has_vlm_desc'):
-                try:
-                    import re
+                        if summary:
+                            ctx['summary'] = summary
+                            ctx['llm_enhanced'] = True
 
+                # 2. 表格有关联图片时，懒加载 VLM 描述
+                if image_path and not meta.get('has_vlm_desc'):
                     # 提取表号（如 "表2.2"、"见表2.1"）
                     table_number = ""
-                    # 匹配 "表2.2"、"见表2.2"、"见表 2.2" 等
                     table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
                     if table_match:
                         table_number = table_match.group(1)
 
-                    # 如果 doc 中没有，尝试从 section 中提取
                     section = meta.get('section') or meta.get('section_path', '')
                     if not table_number and section:
                         table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
                         if table_match:
                             table_number = table_match.group(1)
 
-                    # 构建表格图片元数据
                     table_image_metadata = {
                         'section': section,
                         'page': meta.get('page'),
                         'caption': meta.get('caption', ''),
                         'source': meta.get('source', ''),
-                        'table_number': table_number,  # 表号
-                        'figure_number': table_number,  # 兼容字段
+                        'table_number': table_number,
+                        'figure_number': table_number,
                         'doc_text': doc_text,
-                        'is_table': True  # 标记为表格图片
+                        'is_table': True
                     }
                     vlm_desc = await lazy_vlm_description(
-                        meta.get('id', ''),
+                        meta.get('chunk_id', ''),
                         image_path,
                         kb_name,
-                        metadata=table_image_metadata
+                        metadata=table_image_metadata,
+                        defer_chromadb=defer_chromadb
                     )
-                    # 表格图片描述作为补充信息
-                    ctx['image_description'] = vlm_desc
-                    ctx['vlm_enhanced'] = True
-                except Exception as e:
-                    logger.warning(f"表格图片 VLM 懒加载失败: {e}")
+                    if vlm_desc:
+                        ctx['image_description'] = vlm_desc
+                        ctx['vlm_enhanced'] = True
+
+        except Exception as e:
+            chunk_id = ctx.get('meta', {}).get('chunk_id', '?')
+            logger.warning(f"增强切片失败(chunk_id={chunk_id}): {e}")

knowledge/manager.py

@@ -622,7 +622,7 @@ class KnowledgeBaseManager(
         try:
             from config import get_llm_client, DASHSCOPE_MODEL
             client = get_llm_client()
-            summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
+            summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
             return summary.strip() if summary else ""
         except Exception as e:
             logger.warning(f"生成表格摘要失败: {e}")
@@ -681,13 +681,31 @@ class KnowledgeBaseManager(
                         ]
                     }
                 ],
-                max_tokens=512
+                max_tokens=2048  # mimo-v2.5 推理模型思考链消耗 ~1000 token，需留足输出空间
             )
 
             description = response.choices[0].message.content
 
+            # 推理模型兼容：content 为空时从 reasoning_content 提取
+            if not description or not description.strip():
+                reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
+                if reasoning and reasoning.strip():
+                    import re
+                    # 尝试从思考链中提取有用文本（去掉 <think> 标签后的内容）
+                    cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
+                    if cleaned:
+                        logger.info(f"VLM content为空，从reasoning_content提取描述: {image_path}")
+                        description = cleaned
+                    else:
+                        description = reasoning.strip()
+
+            if not description:
+                logger.warning(f"VLM 返回空描述: {image_path}")
+                return ""
+
             # 缓存结果
             import hashlib
+            import re as _re
             img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
             cache_dir = Path('.data/cache/vlm')
             cache_dir.mkdir(parents=True, exist_ok=True)