diff --git a/knowledge/manager.py b/knowledge/manager.py index dc79c14..150d301 100644 --- a/knowledge/manager.py +++ b/knowledge/manager.py @@ -139,7 +139,7 @@ class KnowledgeBaseManager( logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}") def _load_metadata(self) -> dict: - """加载元数据(带文件锁)""" + """加载元数据(带文件锁,确保多 worker 进程间一致)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) if os.path.exists(metadata_path): try: @@ -156,7 +156,7 @@ class KnowledgeBaseManager( return {"collections": {}} def _save_metadata(self): - """保存元数据(带文件锁,防止并发写入覆盖)""" + """保存元数据(带文件锁,防止并发写入数据覆盖)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) try: with open(metadata_path, 'w', encoding='utf-8') as f: @@ -335,6 +335,7 @@ class KnowledgeBaseManager( "section": section_path, "status": "active", "version": "v1", + "text_level": getattr(chunk, 'text_level', 0), # 标题级别(0=正文,1=h1,2=h2,3=h3),供检索管线层次感知 } if extra_metadata: @@ -347,6 +348,25 @@ class KnowledgeBaseManager( if hasattr(chunk, 'image_path') and chunk.image_path: metadata['image_path'] = chunk.image_path + # bbox 坐标(PDF 有,DOCX 无,需 None 保护) + # _build_citation() 从 metadata 读取 bbox 做引用定位 + chunk_bbox = getattr(chunk, 'bbox', None) + if chunk_bbox: + metadata['bbox'] = json.dumps(chunk_bbox) + + # MinerU 结构化元数据(表格类型、嵌套层级、图片子类型) + chunk_table_type = getattr(chunk, 'table_type', '') + if chunk_table_type: + metadata['table_type'] = chunk_table_type + + chunk_nest_level = getattr(chunk, 'table_nest_level', '') + if chunk_nest_level: + metadata['table_nest_level'] = str(chunk_nest_level) + + chunk_sub_type = getattr(chunk, 'sub_type', '') + if chunk_sub_type: + metadata['sub_type'] = chunk_sub_type + # 生成向量 try: embedding = embedding_model.encode(semantic_content).tolist() @@ -602,7 +622,7 @@ class KnowledgeBaseManager( try: from config import get_llm_client, DASHSCOPE_MODEL client = get_llm_client() - summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048) + summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512) return summary.strip() if summary else "" except Exception as e: logger.warning(f"生成表格摘要失败: {e}") @@ -661,31 +681,13 @@ class KnowledgeBaseManager( ] } ], - max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间 + max_tokens=512 ) description = response.choices[0].message.content - # 推理模型兼容:content 为空时从 reasoning_content 提取 - if not description or not description.strip(): - reasoning = getattr(response.choices[0].message, 'reasoning_content', None) - if reasoning and reasoning.strip(): - import re - # 尝试从思考链中提取有用文本(去掉 标签后的内容) - cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip() - if cleaned: - logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}") - description = cleaned - else: - description = reasoning.strip() - - if not description: - logger.warning(f"VLM 返回空描述: {image_path}") - return "" - # 缓存结果 import hashlib - import re as _re img_hash = hashlib.md5(img_path.read_bytes()).hexdigest() cache_dir = Path('.data/cache/vlm') cache_dir.mkdir(parents=True, exist_ok=True) diff --git a/parsers/heading_rules.py b/parsers/heading_rules.py index 2bba751..23acfe1 100644 --- a/parsers/heading_rules.py +++ b/parsers/heading_rules.py @@ -106,11 +106,12 @@ DEFAULT_HEADING_RULES: List[HeadingRule] = [ name="chinese_chapter", ), # 2. 中文条款编号 -> h2 - # 匹配:第一条、第三款 等 + # 匹配:第一条、第三款 等(仅短标题,长正文段落不算标题) HeadingRule( pattern=re.compile(r'^第[一二三四五六七八九十百千万]+[条款]'), level=2, name="chinese_article", + max_length=30, ), # 3. 数字三级标题 -> h3(必须在二级之前匹配) # 匹配:1.1.1 背景、2.3.4 方案 等 @@ -121,20 +122,23 @@ DEFAULT_HEADING_RULES: List[HeadingRule] = [ max_length=100, ), # 4. 数字二级标题 -> h2(必须在一级之前匹配) - # 匹配:1.1 背景、2.3 方案 等 + # 匹配:1.1 背景、2.3 方案、2.1运行调度(无空格) 等 + # 使用负向前瞻排除三级标题(由 numeric_level3 处理) HeadingRule( - pattern=re.compile(r'^\d+\.\d+[\.、\s]'), + pattern=re.compile(r'^\d+\.\d+(?!\.\d)'), level=2, name="numeric_level2", max_length=80, ), # 5. 数字一级标题 -> h1 # 匹配:1. 概述、2、背景 等 + # 排除:以 ;;。,、: 结尾的文本(这些是编号列表项/子条目,不是独立标题) HeadingRule( pattern=re.compile(r'^\d+[\.、\s]'), level=1, name="numeric_level1", max_length=50, + exclude_pattern=re.compile(r'[;;。,、::]$'), ), # 6. 英文章节标题 -> h1 # 匹配:Chapter 1、Section 2、Part 3 等 @@ -201,6 +205,22 @@ class HeadingRuleEngine: import copy self.rules = copy.deepcopy(DEFAULT_HEADING_RULES) + def _validate_level(self, level: int, text: str, rule_name=None): + """各级别标题长度防护:超长文本不应作为标题,降为正文。 + H1 > 40字, H2 > 60字, H3 > 50字 → 降为正文。 + 统一覆盖 v1 常规匹配、v2 style 匹配、bold_short_text 兜底所有返回路径。""" + text_len = len(text) + if level == 1 and text_len > 40: + logger.debug(f"标题识别: '{text[:30]}...' H1 但超长({text_len}字),降为正文") + return 0, None + if level == 2 and text_len > 60: + logger.debug(f"标题识别: '{text[:30]}...' H2 但超长({text_len}字),降为正文") + return 0, None + if level == 3 and text_len > 50: + logger.debug(f"标题识别: '{text[:30]}...' H3 但超长({text_len}字),降为正文") + return 0, None + return level, rule_name + def detect(self, text: str, style: Optional[List[str]] = None) -> Tuple[int, Optional[str]]: """ 检测文本的标题级别 @@ -227,7 +247,7 @@ class HeadingRuleEngine: level = rule.match(text) if level > 0: logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h{level} (规则: {rule.name})") - return level, rule.name + return self._validate_level(level, text, rule.name) # 再检查是否匹配中文章节/条款等高优先级规则 for rule in self.rules: @@ -236,9 +256,16 @@ class HeadingRuleEngine: level = rule.match(text) if level > 0: logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h{level} (规则: {rule.name})") - return level, rule.name + return self._validate_level(level, text, rule.name) + + # 最后兜底:加粗短文本 → h2 + # 但需先检查所有规则的 exclude_pattern,防止编号列表项被误判为标题 + # 例如 "3.完全满足品规。指..." 虽有 bold 样式,但属于列表项而非标题 + for rule in self.rules: + if rule.enabled and rule.exclude_pattern and rule.exclude_pattern.search(text): + logger.debug(f"标题识别(v2 style): '{text[:30]}' 被 {rule.name} 的 exclude_pattern 排除") + return 0, None - # 否则作为加粗短文本 → h2(与 bold_short_text 规则对齐,但不依赖 **...** 标记) logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h2 (规则: bold_short_text_via_style)") return 2, 'bold_short_text' @@ -247,7 +274,7 @@ class HeadingRuleEngine: level = rule.match(text) if level > 0: logger.debug(f"标题识别: '{text[:30]}' -> h{level} (规则: {rule.name})") - return level, rule.name + return self._validate_level(level, text, rule.name) return 0, None diff --git a/parsers/mineru_parser.py b/parsers/mineru_parser.py index 74845b5..0f34a22 100644 --- a/parsers/mineru_parser.py +++ b/parsers/mineru_parser.py @@ -131,14 +131,21 @@ class MinerUChunk: # 图片上下文(用于语义检索) context_before: str = "" # 图片前的文本上下文 context_after: str = "" # 图片后的文本上下文 + # VLM 增强信息 + vlm_description: str = "" # VLM 视觉描述(图片/图表) + chart_markdown: str = "" # VLM 提取的图表数据表(Markdown 格式) + # MinerU 结构化元数据 + table_type: str = "" # 表格类型(cflow/table/text,来自 _v2_table_type) + table_nest_level: str = "" # 表格嵌套层级(来自 _v2_table_nest_level) + sub_type: str = "" # 图片/图表子类型(natural_image/table_image 等) def parse_with_mineru_online( file_path: str, api_token: str = None, api_url: str = None, - model_version: str = "vlm", - timeout: int = 300 + model_version: str = None, + timeout: int = None ) -> Dict[str, Any]: """ 使用 MinerU 在线 API 解析文档 @@ -157,8 +164,8 @@ def parse_with_mineru_online( file_path: 文档文件路径 api_token: API Token(默认从 config 读取) api_url: API 地址 - model_version: 模型版本 (vlm / pipeline / MinerU-HTML) - timeout: 请求超时(秒) + model_version: 模型版本 (vlm / pipeline / MinerU-HTML),默认从 config 读取 + timeout: 轮询超时(秒),默认从 config 读取 Returns: 解析结果(与 parse_with_mineru 格式相同) @@ -167,10 +174,12 @@ def parse_with_mineru_online( import time import zipfile import io - from config import MINERU_API_TOKEN, MINERU_API_URL + from config import MINERU_API_TOKEN, MINERU_API_URL, MINERU_MODEL_VERSION, MINERU_ONLINE_TIMEOUT token = api_token or MINERU_API_TOKEN url = api_url or MINERU_API_URL + model_version = model_version or MINERU_MODEL_VERSION + timeout = timeout or MINERU_ONLINE_TIMEOUT if not token: raise RuntimeError("MinerU 在线 API Token 未配置,请在 config.py 中设置 MINERU_API_TOKEN") @@ -241,9 +250,15 @@ def parse_with_mineru_online( result_resp.raise_for_status() result = result_resp.json() + # 检查 API 层面的错误码,快速失败而非静默等到超时 + api_code = result.get("code") + if api_code and api_code != 0: + api_msg = result.get("msg", "未知错误") + raise RuntimeError(f"MinerU API 错误 (code={api_code}): {api_msg}") + extract_results = result.get("data", {}).get("extract_result", []) if not extract_results: - logger.debug(f"等待解析结果... ({waited}s)") + logger.debug(f"等待解析结果... ({waited}s/{max_wait}s)") continue # 取第一个文件的结果 @@ -276,11 +291,16 @@ def parse_with_mineru_online( if progress: extracted = progress.get("extracted_pages", 0) total = progress.get("total_pages", 0) - logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s)") + logger.info(f"解析进度: {extracted}/{total} 页 ({waited}s/{max_wait}s, model={model_version})") else: - logger.debug(f"状态: {state}, 等待中... ({waited}s)") + logger.debug(f"状态: {state}, 等待中... ({waited}s/{max_wait}s)") - raise RuntimeError("MinerU 在线解析超时") + raise RuntimeError( + f"MinerU 在线解析超时 ({max_wait}s)" + f",当前 model_version={model_version}" + f",可尝试: 1) 设置 MINERU_MODEL_VERSION=pipeline 加速" + f" 2) 增大 MINERU_ONLINE_TIMEOUT" + ) except Exception as e: raise RuntimeError(f"MinerU 在线 API 调用失败: {e}") @@ -295,15 +315,26 @@ def _parse_v2_content_list(v2_data: list) -> list: 转换规则: - paragraph → text(拼接 paragraph_content,提取 style 信息) - - table → table(保留 table_body/html) - - title → text(提取 level 信息) + - title → text(从 title_content 提取文本,level 信息) + - table → table(保留 html,提取 table_caption/table_footnote 列表格式) + - image → image(提取 image_source.path、VLM 视觉描述 content.content) + - chart → chart(提取 image_source.path、VLM 数据表 content.content Markdown) + - list → text(将 list_items 拼接为段落,保留 list_type) + - equation → equation - page_header / page_footer / page_number → 过滤掉 + 支持的额外字段(_v2_ 前缀): + - _v2_styles: 样式列表(如 ['bold']) + - _v2_table_type / _v2_table_nest_level / _v2_table_footnote: 表格元数据 + - _v2_vlm_description: VLM 生成的图片视觉描述 + - _v2_chart_markdown: VLM 从图表中提取的 Markdown 数据表 + - _v2_list_type: 列表类型(text_list 等) + Args: v2_data: v2 格式的嵌套列表 Returns: - v1 兼容的扁平列表,额外包含 _v2_styles / _v2_table_type 字段 + v1 兼容的扁平列表 """ flat_list: List[Dict] = [] @@ -317,8 +348,8 @@ def _parse_v2_content_list(v2_data: list) -> list: v2_type: str = item.get('type', '') content = item.get('content', {}) - # 过滤噪音类型(页眉、页脚、页码) - if v2_type in ('page_header', 'page_footer', 'page_number'): + # 过滤噪音类型(页眉、页脚、页码、目录索引) + if v2_type in ('page_header', 'page_footer', 'page_number', 'index'): continue if v2_type == 'paragraph': @@ -350,28 +381,68 @@ def _parse_v2_content_list(v2_data: list) -> list: flat_list.append(flat_item) elif v2_type == 'table': + # table_caption 在 V2 中是列表格式: [{"type": "text", "content": "..."}] + table_caption_list = content.get('table_caption', []) if isinstance(content, dict) else [] + table_caption = ''.join( + p.get('content', '') for p in table_caption_list if isinstance(p, dict) + ).strip() if isinstance(table_caption_list, list) else str(table_caption_list) + # 兜底: 如果 caption 列表为空,尝试旧的 caption 字符串字段 + if not table_caption: + table_caption = content.get('caption', '') if isinstance(content, dict) else '' + # table_footnote + table_footnote_list = content.get('table_footnote', []) if isinstance(content, dict) else [] + table_footnote = ''.join( + p.get('content', '') for p in table_footnote_list if isinstance(p, dict) + ).strip() if isinstance(table_footnote_list, list) else '' + # image_source (V2 新格式) 兜底 img_path + img_src = content.get('image_source', {}) if isinstance(content, dict) else {} + img_path = img_src.get('path', '') if isinstance(img_src, dict) else '' + if not img_path: + img_path = content.get('img_path', '') if isinstance(content, dict) else '' + flat_item = { 'type': 'table', 'page_idx': page_idx, 'bbox': item.get('bbox', []), 'html': content.get('html', '') if isinstance(content, dict) else '', 'table_body': content.get('html', '') if isinstance(content, dict) else '', - 'caption': content.get('caption', '') if isinstance(content, dict) else '', + 'caption': table_caption, + 'img_path': img_path, '_v2_table_type': content.get('table_type', '') if isinstance(content, dict) else '', + '_v2_table_nest_level': content.get('table_nest_level', '') if isinstance(content, dict) else '', + '_v2_table_footnote': table_footnote, } flat_list.append(flat_item) elif v2_type == 'title': - # v2 title 项有 level 信息 - level = content.get('level', 0) if isinstance(content, dict) else 0 + # v2 title 项: level 在 content.level,文本在 title_content(非 paragraph_content) + vlm_level = content.get('level', 0) if isinstance(content, dict) else 0 text_parts = [] - para_content = content.get('paragraph_content', []) if isinstance(content, dict) else [] - for part in para_content: + # PDF V2 使用 title_content,DOCX 理论上不应出现 title 类型 + title_content = content.get('title_content', []) if isinstance(content, dict) else [] + # 兜底: 如果 title_content 为空,尝试 paragraph_content + if not title_content: + title_content = content.get('paragraph_content', []) if isinstance(content, dict) else [] + for part in title_content: if isinstance(part, dict): text_parts.append(part.get('content', '')) full_text = ''.join(text_parts).strip() if full_text: + # PDF V2: heading_rules 优先(模式匹配对编号标题可靠) + # VLM level 仅作兜底(VLM 常给所有标题 level=1,不可靠) + level = 0 + try: + from parsers.heading_rules import get_heading_engine + engine = get_heading_engine() + detected_level, rule_name = engine.detect(full_text, style=['bold']) + if detected_level > 0: + level = detected_level + except Exception: + pass + if level == 0 and vlm_level > 0: + level = vlm_level + flat_item = { 'type': 'text', 'text': full_text, @@ -384,16 +455,97 @@ def _parse_v2_content_list(v2_data: list) -> list: flat_list.append(flat_item) elif v2_type in ('image', 'chart'): + # image_source (V2 新格式) 兜底 img_path + img_src = content.get('image_source', {}) if isinstance(content, dict) else {} + img_path = img_src.get('path', '') if isinstance(img_src, dict) else '' + if not img_path: + img_path = content.get('img_path', '') if isinstance(content, dict) else '' + + # caption 在 V2 中可能是列表格式 + caption_raw = content.get('image_caption', content.get('caption', '')) if isinstance(content, dict) else '' + if isinstance(caption_raw, list): + caption = ''.join(p.get('content', '') for p in caption_raw if isinstance(p, dict)).strip() + else: + caption = str(caption_raw) + + # VLM 视觉描述 (image 和 chart 项的 content.content 字段) + vlm_description = '' + if isinstance(content, dict): + desc = content.get('content', '') + if isinstance(desc, str) and desc and len(desc) > 10: + # image 直接使用;chart 需排除 markdown 表格(表格走 chart_markdown) + if v2_type == 'image': + vlm_description = desc + elif v2_type == 'chart' and '|' not in desc: + vlm_description = desc + + # chart 的 VLM 数据表 (content.content 字段,Markdown 表格) + chart_markdown = '' + if v2_type == 'chart' and isinstance(content, dict): + md = content.get('content', '') + if isinstance(md, str) and md: + if '|' in md: + chart_markdown = md + # 即使没有 '|',如果有结构化数据特征也保留 + elif len(md) > 50 and any(kw in md for kw in ('数据', '合计', '总计', '年份', '单位')): + chart_markdown = md + # chart caption + chart_caption_list = content.get('chart_caption', []) + if isinstance(chart_caption_list, list) and chart_caption_list: + caption = ''.join( + p.get('content', '') for p in chart_caption_list if isinstance(p, dict) + ).strip() or caption + + # sub_type (natural_image / table_image 等) + sub_type = item.get('sub_type', '') + + # 封面 logo 过滤:第一页无 caption 无 VLM 描述的图片通常是封面装饰 + if (v2_type == 'image' and page_idx == 0 + and not caption and not vlm_description and not img_path): + continue + flat_item = { 'type': v2_type, 'page_idx': page_idx, 'bbox': item.get('bbox', []), - 'img_path': content.get('img_path', '') if isinstance(content, dict) else '', - 'image_path': content.get('img_path', '') if isinstance(content, dict) else '', - 'caption': content.get('caption', '') if isinstance(content, dict) else '', + 'img_path': img_path, + 'image_path': img_path, + 'caption': caption, + 'sub_type': sub_type, + '_v2_vlm_description': vlm_description, + '_v2_chart_markdown': chart_markdown, } flat_list.append(flat_item) + elif v2_type == 'list': + # 结构化列表:将列表项拼接为段落文本 + list_items = content.get('list_items', []) if isinstance(content, dict) else [] + item_texts = [] + for li in list_items: + if not isinstance(li, dict): + continue + item_content = li.get('item_content', []) + text = ''.join( + p.get('content', '') for p in item_content if isinstance(p, dict) + ).strip() + if text: + item_texts.append(text) + + if item_texts: + list_type = content.get('list_type', 'text_list') if isinstance(content, dict) else 'text_list' + full_text = '\n'.join(item_texts) + flat_item = { + 'type': 'text', + 'text': full_text, + 'content': full_text, + 'page_idx': page_idx, + 'bbox': item.get('bbox', []), + 'text_level': 0, + '_v2_styles': [], + '_v2_list_type': list_type, + } + flat_list.append(flat_item) + elif v2_type == 'equation': flat_item = { 'type': 'equation', @@ -406,6 +558,100 @@ def _parse_v2_content_list(v2_data: list) -> list: } flat_list.append(flat_item) + # ── TOC 残留过滤 ── + # 目录条目可能以 list / title / paragraph 类型混入,用行尾页码模式检测 + # 模式1: 连续点号/省略号+页码(如 "1 综述..........1"、"2 三峡工程…4") + # 模式2: 短行+空格+页码数字(如 "2.2 防洪 8"、"6.2 水位 28") + import re + _toc_dots = re.compile(r'(\.{2,}|…+|⋯+)\s*\d+\s*$') + _toc_space_num = re.compile(r'\s{2,}\d{1,3}\s*$') # 2+空格+1-3位数字 + _toc_filtered = 0 + filtered_list = [] + for fi in flat_list: + text = fi.get('text', '') or fi.get('content', '') + # 只检查 text 类型(title/list/paragraph 产出),table/image/chart 不动 + if fi.get('type') == 'text' and text: + lines = [l.strip() for l in text.split('\n') if l.strip()] + if lines: + toc_hits = 0 + for l in lines: + if _toc_dots.search(l): + toc_hits += 1 + elif _toc_space_num.search(l) and len(l) < 40: + # 短行+空格+页码:典型的目录格式 + toc_hits += 1 + # TOC 判定逻辑: + # 1) 匹配率 > 50% → 确定是目录 + # 2) 匹配率 > 25% 且平均行长 < 25字 → 目录(短行+页码是强信号) + avg_line_len = sum(len(l) for l in lines) / len(lines) if lines else 0 + match_ratio = toc_hits / len(lines) if lines else 0 + is_toc = (match_ratio > 0.5) or (match_ratio > 0.25 and avg_line_len < 25) + if is_toc: + _toc_filtered += 1 + logger.debug(f"TOC 过滤命中({toc_hits}/{len(lines)}, avg={avg_line_len:.0f}): {text[:60]}...") + continue + elif toc_hits > 0: + logger.debug(f"TOC 部分匹配({toc_hits}/{len(lines)}): {text[:60]}...") + filtered_list.append(fi) + if _toc_filtered > 0: + logger.info(f"TOC 过滤第一轮: 移除 {_toc_filtered} 个目录块") + flat_list = filtered_list + + # 第二轮:清理孤立的 TOC 标题(子条目被过滤后残留的父标题) + _toc_orphan = 0 + final_list = [] + for fi in flat_list: + text = (fi.get('text', '') or fi.get('content', '')).strip() + page = fi.get('page_idx', 0) + if fi.get('type') == 'text' and text and page <= 5: + # 孤立 "目录" 标题 + if text == '目录': + _toc_orphan += 1 + continue + # 短标题 + 尾部页码数字(如 "6 长江中下游河道状况 25") + if (len(text) < 40 and not text.endswith('。') and not text.endswith(';') + and re.search(r'\s+\d{1,3}\s*$', text)): + _toc_orphan += 1 + continue + final_list.append(fi) + if _toc_orphan > 0: + logger.info(f"TOC 过滤第二轮: 移除 {_toc_orphan} 个孤立目录标题") + flat_list = final_list + + # ── 单字符标题残留过滤 ── + # VLM 可能部分识别目录标题(如 "目录" → "录"),单字符标题几乎不会是有效章节 + _single_char = 0 + _sc_list = [] + for fi in flat_list: + text = (fi.get('text', '') or fi.get('content', '')).strip() + if fi.get('type') == 'text' and text and len(text) <= 1 and fi.get('text_level', 0) > 0: + _single_char += 1 + logger.debug(f"单字符标题过滤: '{text}' pg={fi.get('page_idx', 0)}") + continue + _sc_list.append(fi) + if _single_char > 0: + logger.info(f"单字符标题过滤: 移除 {_single_char} 个残留") + flat_list = _sc_list + + # ── 封面重复标题去重 ── + # pg=0(封面)和 pg=1(扉页)常有完全相同的标题(如 "三峡工程公报"、"2022"),保留较后的 + _cover_seen = {} # text -> page_idx + _cover_dup = 0 + dedup_list = [] + for fi in flat_list: + text = (fi.get('text', '') or fi.get('content', '')).strip() + page = fi.get('page_idx', 0) + if fi.get('type') == 'text' and text and page <= 1 and fi.get('text_level', 0) > 0: + if text in _cover_seen: + _cover_dup += 1 + logger.debug(f"封面重复标题过滤: '{text}' pg={page} (首次 pg={_cover_seen[text]})") + continue + _cover_seen[text] = page + dedup_list.append(fi) + if _cover_dup > 0: + logger.info(f"封面去重: 移除 {_cover_dup} 个重复封面标题") + flat_list = dedup_list + logger.info(f"v2 格式转换: {len(v2_data)} 页 → {len(flat_list)} 项(已过滤噪音类型)") return flat_list @@ -615,6 +861,7 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any] elif item_type == "table": table_body = item.get("html", "") or item.get("table_body", "") table_caption = item.get("caption", "") or item.get("table_caption", "") + table_footnote = item.get("_v2_table_footnote", "") img_path = item.get("img_path", "") or item.get("image_path", "") section_path = " > ".join([s[1] for s in section_stack]) @@ -629,8 +876,13 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any] # 从原始 HTML 提取嵌入图片(md_table 经 get_text 转换后已丢失 标签) table_images = extract_images_from_markdown(table_body) if table_body else [] + # 表格内容增强:caption + footnote + table_content = table_caption or "表格" + if table_footnote: + table_content = f"{table_content}\n[脚注] {table_footnote}" + chunk = MinerUChunk( - content=table_caption or "表格", + content=table_content, chunk_type="table", page_start=page_idx + 1, page_end=page_idx + 1, @@ -640,7 +892,9 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any] source_file=file_path.name, table_html=table_body, image_path=img_path, - images=table_images if table_images else None + images=table_images if table_images else None, + table_type=item.get("_v2_table_type", ""), + table_nest_level=item.get("_v2_table_nest_level", ""), ) chunks.append(chunk) if table_body: @@ -651,16 +905,29 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any] elif item_type in ("image", "chart"): img_path = item.get("img_path", "") or item.get("image_path", "") caption = item.get("caption", "") + vlm_desc = item.get("_v2_vlm_description", "") + chart_md = item.get("_v2_chart_markdown", "") + sub_type = item.get("sub_type", "") section_path = " > ".join([s[1] for s in section_stack]) markdown_parts.append(f"\n![{caption}]({img_path})") + # chart 的 VLM 数据表也写入 markdown 输出 + if chart_md: + markdown_parts.append(chart_md) chunk_type = "chart" if item_type == "chart" else "image" context_before, context_after = get_context_for_image(idx, page_idx) + # 图片内容增强:caption + VLM 描述 + content_text = caption or ("图表" if item_type == "chart" else "图片") + if vlm_desc: + content_text = f"{content_text}\n[视觉描述] {vlm_desc}" + if chart_md: + content_text = f"{content_text}\n[数据表]\n{chart_md}" + chunk = MinerUChunk( - content=caption or ("图表" if item_type == "chart" else "图片"), + content=content_text, chunk_type=chunk_type, page_start=page_idx + 1, page_end=page_idx + 1, @@ -670,11 +937,18 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any] source_file=file_path.name, image_path=img_path, context_before=context_before, - context_after=context_after + context_after=context_after, + vlm_description=vlm_desc, + chart_markdown=chart_md, + table_html=chart_md if chart_md else None, # chart 数据表作为表格存储 + sub_type=sub_type, ) chunks.append(chunk) if img_path: images.append(img_path) + # chart 数据表也加入 tables 列表,便于表格检索 + if chart_md: + tables.append(chart_md) elif item_type == "equation": # 处理公式类型 @@ -737,7 +1011,7 @@ def parse_with_mineru( lang: str = "ch", enable_table: bool = True, enable_formula: bool = True, - backend: str = "pipeline", + backend: str = None, start_page: int = 0, end_page: int = 99999 ) -> Dict[str, Any]: @@ -770,6 +1044,14 @@ def parse_with_mineru( if not file_path.exists(): raise FileNotFoundError(f"文件不存在: {file_path}") + # 从 config 读取默认 backend + if backend is None: + try: + from config import MINERU_LOCAL_BACKEND + backend = MINERU_LOCAL_BACKEND + except ImportError: + backend = 'pipeline' + # 检查文件大小 file_size = file_path.stat().st_size if file_size > MAX_PDF_SIZE: @@ -1127,6 +1409,7 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: elif item_type == "table": table_body = item.get("table_body", "") table_caption = item.get("table_caption", "") + table_footnote = item.get("_v2_table_footnote", "") # 表格也可能有图片形式(img_path) img_path = item.get("img_path", "") @@ -1142,8 +1425,13 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: # 从原始 HTML 提取嵌入图片(md_table 经 get_text 转换后已丢失 标签) table_images = extract_images_from_markdown(table_body) if table_body else [] + # 表格内容增强:caption + footnote + table_content = table_caption or "表格" + if table_footnote: + table_content = f"{table_content}\n[脚注] {table_footnote}" + chunk = MinerUChunk( - content=table_caption or "表格", + content=table_content, chunk_type="table", page_start=page_idx + 1, page_end=page_idx + 1, @@ -1153,7 +1441,9 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: source_file=file_path.name, table_html=table_body, image_path=img_path, # 表格的独立图片形式 - images=table_images if table_images else None # 嵌入图片列表 + images=table_images if table_images else None, # 嵌入图片列表 + table_type=item.get("_v2_table_type", ""), + table_nest_level=item.get("_v2_table_nest_level", ""), ) chunks.append(chunk) if table_body: @@ -1166,10 +1456,15 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: # 处理图片和图表类型(MinerU 将图表识别为 chart 类型) img_path = item.get("img_path", "") caption = item.get("caption", "") + vlm_desc = item.get("_v2_vlm_description", "") + chart_md = item.get("_v2_chart_markdown", "") + sub_type = item.get("sub_type", "") section_path = " > ".join([s[1] for s in section_stack]) markdown_parts.append(f"\n![{caption}]({img_path})") + if chart_md: + markdown_parts.append(chart_md) # 图表类型标记为 chart,便于后续区分处理 chunk_type = "chart" if item_type == "chart" else "image" @@ -1177,8 +1472,15 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: # 获取图片上下文 context_before, context_after = get_context_for_image(idx, page_idx) + # 图片内容增强:caption + VLM 描述 + content_text = caption or ("图表" if item_type == "chart" else "图片") + if vlm_desc: + content_text = f"{content_text}\n[视觉描述] {vlm_desc}" + if chart_md: + content_text = f"{content_text}\n[数据表]\n{chart_md}" + chunk = MinerUChunk( - content=caption or ("图表" if item_type == "chart" else "图片"), + content=content_text, chunk_type=chunk_type, page_start=page_idx + 1, page_end=page_idx + 1, @@ -1188,11 +1490,17 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]: source_file=file_path.name, image_path=img_path, context_before=context_before, - context_after=context_after + context_after=context_after, + vlm_description=vlm_desc, + chart_markdown=chart_md, + table_html=chart_md if chart_md else None, + sub_type=sub_type, ) chunks.append(chunk) if img_path: images.append(img_path) + if chart_md: + tables.append(chart_md) elif item_type == "equation": # 处理公式类型 @@ -1301,6 +1609,7 @@ def _post_process_chunks( # Phase 2: 合并碎片 merged = [] buffer = None # 当前合并缓冲 + _buffer_has_body = False # 缓冲是否已包含正文(防止标题继续合并) for chunk in chunks: # 表格、图片和图表不参与合并,直接输出 @@ -1308,13 +1617,34 @@ def _post_process_chunks( if buffer: merged.append(buffer) buffer = None + _buffer_has_body = False merged.append(chunk) continue - # 标题 chunk(text_level > 0),开始新的合并组 + # 标题 chunk(text_level > 0) if chunk.text_level > 0: if buffer: - merged.append(buffer) + # H1 级标题是章节边界,强制断开,不参与连续标题链合并 + if chunk.text_level == 1: + merged.append(buffer) + _buffer_has_body = False + # 连续标题链合并:缓冲也是纯标题(无正文)时,合并而非刷新(仅非 H1) + elif buffer.text_level > 0 and not _buffer_has_body: + combined = buffer.content.rstrip() + '\n' + chunk.content + if len(combined) <= max_merged_size: + buffer.content = combined + buffer.page_end = chunk.page_end + # 取更高层级(数值更小) + buffer.text_level = min(buffer.text_level, chunk.text_level) + # section_path 保留第一个(更高级别)的 + continue + # 合并后超限,刷新缓冲 + merged.append(buffer) + _buffer_has_body = False + else: + # 缓冲是正文,正常刷新 + merged.append(buffer) + _buffer_has_body = False # 标题作为新缓冲的起点 buffer = MinerUChunk( content=chunk.content, @@ -1327,6 +1657,7 @@ def _post_process_chunks( bbox=chunk.bbox, source_file=chunk.source_file, ) + _buffer_has_body = False continue # 正文 chunk @@ -1346,6 +1677,7 @@ def _post_process_chunks( bbox=chunk.bbox, source_file=chunk.source_file, ) + _buffer_has_body = True # 正文 chunk 创建的缓冲已含正文 else: # 足够长,直接输出 merged.append(chunk) @@ -1356,6 +1688,9 @@ def _post_process_chunks( # 合并 buffer.content = buffer.content.rstrip() + '\n' + chunk.content buffer.page_end = chunk.page_end + # 正文并入标题缓冲后,标记已含正文,防止后续标题继续合并 + # 保留 text_level 不置零,使标题层级信息传递到向量库 + _buffer_has_body = True else: # 超过上限,输出缓冲,当前 chunk 开始新缓冲或直接输出 merged.append(buffer) @@ -1371,8 +1706,10 @@ def _post_process_chunks( bbox=chunk.bbox, source_file=chunk.source_file, ) + _buffer_has_body = True # 正文 chunk 创建的缓冲已含正文 else: buffer = None + _buffer_has_body = False merged.append(chunk) # 刷新最后的缓冲 @@ -1687,7 +2024,7 @@ def parse_with_mineru_persistent( lang: str = "ch", enable_table: bool = True, enable_formula: bool = True, - backend: str = "pipeline", + backend: str = None, start_page: int = 0, end_page: int = 99999, cleanup_after_image_move: bool = True @@ -1726,6 +2063,14 @@ def parse_with_mineru_persistent( if not file_path.exists(): raise FileNotFoundError(f"文件不存在: {file_path}") + # 从 config 读取默认 backend + if backend is None: + try: + from config import MINERU_LOCAL_BACKEND + backend = MINERU_LOCAL_BACKEND + except ImportError: + backend = 'pipeline' + # 检查文件大小 file_size = file_path.stat().st_size if file_size > MAX_PDF_SIZE: