feat(server-release): 从 main 同步切片层级结构修复与解析增强

- parsers/mineru_parser.py: MinerU V2 解析增强 + 切片层级结构修复
- parsers/heading_rules.py: 标题识别规则引擎增强
- knowledge/manager.py: 跨页表格合并与切片后处理优化

切片数从旧版 819 降至 437(与本地 VLM 解析的 448 接近)
This commit is contained in:
lacerate551
2026-06-20 22:00:57 +08:00
parent 8d60616124
commit 2c84cb8a5f
3 changed files with 437 additions and 63 deletions

View File

@@ -139,7 +139,7 @@ class KnowledgeBaseManager(
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
def _load_metadata(self) -> dict:
"""加载元数据(带文件锁)"""
"""加载元数据(带文件锁,确保多 worker 进程间一致"""
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
if os.path.exists(metadata_path):
try:
@@ -156,7 +156,7 @@ class KnowledgeBaseManager(
return {"collections": {}}
def _save_metadata(self):
"""保存元数据(带文件锁,防止并发写入覆盖)"""
"""保存元数据(带文件锁,防止并发写入数据覆盖)"""
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
try:
with open(metadata_path, 'w', encoding='utf-8') as f:
@@ -335,6 +335,7 @@ class KnowledgeBaseManager(
"section": section_path,
"status": "active",
"version": "v1",
"text_level": getattr(chunk, 'text_level', 0), # 标题级别(0=正文,1=h1,2=h2,3=h3),供检索管线层次感知
}
if extra_metadata:
@@ -347,6 +348,25 @@ class KnowledgeBaseManager(
if hasattr(chunk, 'image_path') and chunk.image_path:
metadata['image_path'] = chunk.image_path
# bbox 坐标PDF 有DOCX 无,需 None 保护)
# _build_citation() 从 metadata 读取 bbox 做引用定位
chunk_bbox = getattr(chunk, 'bbox', None)
if chunk_bbox:
metadata['bbox'] = json.dumps(chunk_bbox)
# MinerU 结构化元数据(表格类型、嵌套层级、图片子类型)
chunk_table_type = getattr(chunk, 'table_type', '')
if chunk_table_type:
metadata['table_type'] = chunk_table_type
chunk_nest_level = getattr(chunk, 'table_nest_level', '')
if chunk_nest_level:
metadata['table_nest_level'] = str(chunk_nest_level)
chunk_sub_type = getattr(chunk, 'sub_type', '')
if chunk_sub_type:
metadata['sub_type'] = chunk_sub_type
# 生成向量
try:
embedding = embedding_model.encode(semantic_content).tolist()
@@ -602,7 +622,7 @@ class KnowledgeBaseManager(
try:
from config import get_llm_client, DASHSCOPE_MODEL
client = get_llm_client()
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
return summary.strip() if summary else ""
except Exception as e:
logger.warning(f"生成表格摘要失败: {e}")
@@ -661,31 +681,13 @@ class KnowledgeBaseManager(
]
}
],
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token需留足输出空间
max_tokens=512
)
description = response.choices[0].message.content
# 推理模型兼容content 为空时从 reasoning_content 提取
if not description or not description.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip():
import re
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if cleaned:
logger.info(f"VLM content为空从reasoning_content提取描述: {image_path}")
description = cleaned
else:
description = reasoning.strip()
if not description:
logger.warning(f"VLM 返回空描述: {image_path}")
return ""
# 缓存结果
import hashlib
import re as _re
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
cache_dir = Path('.data/cache/vlm')
cache_dir.mkdir(parents=True, exist_ok=True)