feat(server-release): 从 main 同步切片层级结构修复与解析增强
- parsers/mineru_parser.py: MinerU V2 解析增强 + 切片层级结构修复 - parsers/heading_rules.py: 标题识别规则引擎增强 - knowledge/manager.py: 跨页表格合并与切片后处理优化 切片数从旧版 819 降至 437(与本地 VLM 解析的 448 接近)
This commit is contained in:
@@ -139,7 +139,7 @@ class KnowledgeBaseManager(
|
||||
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
||||
|
||||
def _load_metadata(self) -> dict:
|
||||
"""加载元数据(带文件锁)"""
|
||||
"""加载元数据(带文件锁,确保多 worker 进程间一致)"""
|
||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||
if os.path.exists(metadata_path):
|
||||
try:
|
||||
@@ -156,7 +156,7 @@ class KnowledgeBaseManager(
|
||||
return {"collections": {}}
|
||||
|
||||
def _save_metadata(self):
|
||||
"""保存元数据(带文件锁,防止并发写入覆盖)"""
|
||||
"""保存元数据(带文件锁,防止并发写入数据覆盖)"""
|
||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||
try:
|
||||
with open(metadata_path, 'w', encoding='utf-8') as f:
|
||||
@@ -335,6 +335,7 @@ class KnowledgeBaseManager(
|
||||
"section": section_path,
|
||||
"status": "active",
|
||||
"version": "v1",
|
||||
"text_level": getattr(chunk, 'text_level', 0), # 标题级别(0=正文,1=h1,2=h2,3=h3),供检索管线层次感知
|
||||
}
|
||||
|
||||
if extra_metadata:
|
||||
@@ -347,6 +348,25 @@ class KnowledgeBaseManager(
|
||||
if hasattr(chunk, 'image_path') and chunk.image_path:
|
||||
metadata['image_path'] = chunk.image_path
|
||||
|
||||
# bbox 坐标(PDF 有,DOCX 无,需 None 保护)
|
||||
# _build_citation() 从 metadata 读取 bbox 做引用定位
|
||||
chunk_bbox = getattr(chunk, 'bbox', None)
|
||||
if chunk_bbox:
|
||||
metadata['bbox'] = json.dumps(chunk_bbox)
|
||||
|
||||
# MinerU 结构化元数据(表格类型、嵌套层级、图片子类型)
|
||||
chunk_table_type = getattr(chunk, 'table_type', '')
|
||||
if chunk_table_type:
|
||||
metadata['table_type'] = chunk_table_type
|
||||
|
||||
chunk_nest_level = getattr(chunk, 'table_nest_level', '')
|
||||
if chunk_nest_level:
|
||||
metadata['table_nest_level'] = str(chunk_nest_level)
|
||||
|
||||
chunk_sub_type = getattr(chunk, 'sub_type', '')
|
||||
if chunk_sub_type:
|
||||
metadata['sub_type'] = chunk_sub_type
|
||||
|
||||
# 生成向量
|
||||
try:
|
||||
embedding = embedding_model.encode(semantic_content).tolist()
|
||||
@@ -602,7 +622,7 @@ class KnowledgeBaseManager(
|
||||
try:
|
||||
from config import get_llm_client, DASHSCOPE_MODEL
|
||||
client = get_llm_client()
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
||||
return summary.strip() if summary else ""
|
||||
except Exception as e:
|
||||
logger.warning(f"生成表格摘要失败: {e}")
|
||||
@@ -661,31 +681,13 @@ class KnowledgeBaseManager(
|
||||
]
|
||||
}
|
||||
],
|
||||
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
description = response.choices[0].message.content
|
||||
|
||||
# 推理模型兼容:content 为空时从 reasoning_content 提取
|
||||
if not description or not description.strip():
|
||||
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||
if reasoning and reasoning.strip():
|
||||
import re
|
||||
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
|
||||
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||
if cleaned:
|
||||
logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}")
|
||||
description = cleaned
|
||||
else:
|
||||
description = reasoning.strip()
|
||||
|
||||
if not description:
|
||||
logger.warning(f"VLM 返回空描述: {image_path}")
|
||||
return ""
|
||||
|
||||
# 缓存结果
|
||||
import hashlib
|
||||
import re as _re
|
||||
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
||||
cache_dir = Path('.data/cache/vlm')
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
Reference in New Issue
Block a user