Compare commits
5 Commits
183a57e7f1
...
db887d2215
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
db887d2215 | ||
|
|
3c262b8d35 | ||
|
|
8adb550931 | ||
|
|
9c1593a5e4 | ||
|
|
83884b383b |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -120,8 +120,9 @@ test_*.json
|
||||
rag_response.json
|
||||
nul
|
||||
|
||||
# 临时调试脚本(下划线开头)
|
||||
scripts/_*.py
|
||||
# 调试脚本和临时计划(仅本地使用)
|
||||
scripts/
|
||||
plans/
|
||||
|
||||
# Qoder 工具目录
|
||||
.qoder/
|
||||
|
||||
@@ -12,7 +12,8 @@ from flask import Blueprint, request, jsonify
|
||||
from auth.gateway import require_gateway_auth, require_role, get_user_permissions, MOCK_USERS
|
||||
from core.status_codes import SUCCESS, BAD_REQUEST, UNAUTHORIZED, FORBIDDEN, PERMISSION_DENIED, INTERNAL_ERROR
|
||||
from api.response_utils import success_response, error_response
|
||||
import os
|
||||
from config import DEV_MODE
|
||||
import time
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
@@ -23,6 +24,37 @@ load_dotenv(env_path)
|
||||
auth_bp = Blueprint('auth', __name__)
|
||||
|
||||
|
||||
# ==================== 登录速率限制 ====================
|
||||
# 简单的内存级速率限制:每个 IP 在时间窗口内最多允许 N 次登录尝试
|
||||
_login_attempts = {} # {ip: [(timestamp, ...), ...]}
|
||||
_RATE_LIMIT_WINDOW = 300 # 5 分钟窗口
|
||||
_RATE_LIMIT_MAX = 10 # 窗口内最多 10 次尝试
|
||||
|
||||
|
||||
def _check_rate_limit(client_ip: str) -> bool:
|
||||
"""检查是否超出登录速率限制,返回 True 表示允许"""
|
||||
now = time.time()
|
||||
if client_ip not in _login_attempts:
|
||||
_login_attempts[client_ip] = []
|
||||
|
||||
# 清理过期记录
|
||||
_login_attempts[client_ip] = [
|
||||
t for t in _login_attempts[client_ip]
|
||||
if now - t < _RATE_LIMIT_WINDOW
|
||||
]
|
||||
|
||||
if len(_login_attempts[client_ip]) >= _RATE_LIMIT_MAX:
|
||||
return False
|
||||
|
||||
_login_attempts[client_ip].append(now)
|
||||
return True
|
||||
|
||||
|
||||
def _is_dev_mode() -> bool:
|
||||
"""统一的开发模式判断(由 config.py 集中管理)"""
|
||||
return DEV_MODE
|
||||
|
||||
|
||||
@auth_bp.route('/auth/login', methods=['POST'])
|
||||
def mock_login():
|
||||
"""
|
||||
@@ -52,9 +84,17 @@ def mock_login():
|
||||
- manager / manager123 (经理,财务部)
|
||||
- user / test123 (普通用户,技术部)
|
||||
"""
|
||||
# 默认开启开发模式(生产环境需设置 DEV_MODE=false)
|
||||
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用,请设置 DEV_MODE=true", http_status=403)
|
||||
if not _is_dev_mode():
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用,请在 .env 中设置 DEV_MODE=true", http_status=403)
|
||||
|
||||
# 速率限制检查
|
||||
client_ip = request.remote_addr or 'unknown'
|
||||
if not _check_rate_limit(client_ip):
|
||||
return error_response(
|
||||
"RATE_LIMITED", FORBIDDEN,
|
||||
f"登录尝试过于频繁,请 {_RATE_LIMIT_WINDOW // 60} 分钟后再试",
|
||||
http_status=429
|
||||
)
|
||||
|
||||
data = request.json or {}
|
||||
username = data.get('username')
|
||||
@@ -81,7 +121,9 @@ def mock_login():
|
||||
def get_stats():
|
||||
"""获取系统统计信息(仅管理员)"""
|
||||
from flask import current_app
|
||||
session_manager = current_app.config['SESSION_MANAGER']
|
||||
session_manager = current_app.config.get('SESSION_MANAGER')
|
||||
if not session_manager:
|
||||
return error_response("UNAVAILABLE", INTERNAL_ERROR, "会话管理器未启用", http_status=503)
|
||||
return success_response(data=session_manager.get_stats())
|
||||
|
||||
|
||||
@@ -133,8 +175,7 @@ def get_users():
|
||||
]
|
||||
}
|
||||
"""
|
||||
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||
if not dev_mode:
|
||||
if not _is_dev_mode():
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
|
||||
|
||||
users = []
|
||||
@@ -161,12 +202,26 @@ def update_user(user_id):
|
||||
"is_active": false
|
||||
}
|
||||
"""
|
||||
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||
if not dev_mode:
|
||||
if not _is_dev_mode():
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
|
||||
|
||||
# 模拟用户不支持真正的状态切换,直接返回成功
|
||||
return success_response(data={"message": "操作成功(模拟)", "user_id": user_id})
|
||||
# 验证目标用户是否存在
|
||||
target_user = None
|
||||
for username, info in MOCK_USERS.items():
|
||||
if info['user_id'] == user_id:
|
||||
target_user = info
|
||||
break
|
||||
|
||||
if not target_user:
|
||||
return error_response("NOT_FOUND", BAD_REQUEST, f"用户 {user_id} 不存在", http_status=404)
|
||||
|
||||
data = request.json or {}
|
||||
# 模拟操作:记录请求但不实际执行(mock 用户数据是静态的)
|
||||
return success_response(data={
|
||||
"message": "操作成功(模拟)",
|
||||
"user_id": user_id,
|
||||
"applied_changes": data
|
||||
})
|
||||
|
||||
|
||||
@auth_bp.route('/auth/change-password', methods=['POST'])
|
||||
@@ -181,8 +236,7 @@ def change_password():
|
||||
"new_password": "xxx"
|
||||
}
|
||||
"""
|
||||
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||
if not dev_mode:
|
||||
if not _is_dev_mode():
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
|
||||
|
||||
data = request.json or {}
|
||||
@@ -195,5 +249,12 @@ def change_password():
|
||||
if len(new_password) < 6:
|
||||
return error_response("INVALID_PARAMS", BAD_REQUEST, "新密码至少6位", http_status=400)
|
||||
|
||||
# 模拟环境直接返回成功
|
||||
# 验证当前用户的旧密码
|
||||
user = request.current_user
|
||||
username = user.get('username', '')
|
||||
mock_user = MOCK_USERS.get(username)
|
||||
if mock_user and mock_user['password'] != old_password:
|
||||
return error_response("UNAUTHORIZED", UNAUTHORIZED, "旧密码错误", http_status=401)
|
||||
|
||||
# 模拟环境返回成功(不实际修改密码,mock 数据是静态的)
|
||||
return success_response(message="密码修改成功(模拟)")
|
||||
|
||||
@@ -188,6 +188,84 @@ def _get_full_table_from_docstore(chunk_id: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def _strip_semantic_prefix(doc: str, chunk_type: str) -> str:
|
||||
"""
|
||||
去除切片 doc 中的冗余语义前缀,保留关键标识信息
|
||||
|
||||
表格切片的 doc 由 _build_semantic_content_for_table 生成,格式为:
|
||||
主题:section_path(保留,用于关联章节)
|
||||
字段:A, B, C(去除,冗余)
|
||||
描述:该表包含N行数据(去除,冗余)
|
||||
示例:字段=值(去除,冗余)
|
||||
|
||||
表格内容:
|
||||
| A | B | C |
|
||||
|---|---|---|
|
||||
| 1 | 2 | 3 |
|
||||
|
||||
优化策略:
|
||||
- 保留"主题:"行(表格所属章节标识)
|
||||
- 去除"字段:"/"描述:"/"示例:"行(冗余信息)
|
||||
- 添加"【表格】"标记,让 LLM 明确识别表格类型
|
||||
- 保留 Markdown 表格内容(| 开头)和 HTML 表格内容(<table/<tr 等)
|
||||
|
||||
Args:
|
||||
doc: 原始 doc 内容
|
||||
chunk_type: 切片类型
|
||||
|
||||
Returns:
|
||||
精简后的 doc 内容
|
||||
"""
|
||||
if chunk_type != 'table' or not doc:
|
||||
return doc
|
||||
|
||||
import re
|
||||
_html_table_re = re.compile(r'^\s*<\s*(table|tr|td|th|tbody|thead|caption)', re.IGNORECASE)
|
||||
|
||||
lines = doc.split('\n')
|
||||
result_lines = []
|
||||
in_table_section = False
|
||||
theme_line = None
|
||||
|
||||
for line in lines:
|
||||
# 保留"主题:"行(表格所属章节标识)
|
||||
if line.startswith('主题:'):
|
||||
theme_line = line
|
||||
continue
|
||||
|
||||
# 跳过冗余的语义增强行
|
||||
if line.startswith('字段:') or line.startswith('描述:') or line.startswith('示例:'):
|
||||
continue
|
||||
|
||||
# 遇到"表格内容:"标记,进入表格区域
|
||||
if line.strip() == '表格内容:':
|
||||
in_table_section = True
|
||||
continue
|
||||
|
||||
# 表格区域的内容直接保留
|
||||
if in_table_section:
|
||||
result_lines.append(line)
|
||||
elif line.startswith('|'):
|
||||
# 没有"表格内容:"标记时,直接遇到 Markdown 表格行
|
||||
in_table_section = True
|
||||
result_lines.append(line)
|
||||
elif _html_table_re.match(line):
|
||||
# 没有"表格内容:"标记时,直接遇到 HTML 表格标签
|
||||
in_table_section = True
|
||||
result_lines.append(line)
|
||||
|
||||
# 构建结果:主题行 + 【表格】标记 + 表格内容
|
||||
if result_lines:
|
||||
output_parts = []
|
||||
if theme_line:
|
||||
output_parts.append(theme_line)
|
||||
output_parts.append('【表格】')
|
||||
output_parts.extend(result_lines)
|
||||
return '\n'.join(output_parts)
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks: int,
|
||||
min_score: float = 0.0) -> List[Dict]:
|
||||
"""
|
||||
@@ -248,8 +326,31 @@ def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks
|
||||
})
|
||||
|
||||
# Phase 1:按 Rerank 分数过滤低分切片
|
||||
# 保护策略:同一 section 内如有切片通过阈值,则同 section 的 table 切片也保留
|
||||
# 原因:表格切片的 rerank 分数往往偏低(尤其是元问题如"有表格吗?"),
|
||||
# 但它们与同 section 的 text 切片属于同一语义单元,不应割裂
|
||||
# 安全下限:被保护的 table 切片自身 score 不得低于 min_score * 0.3,
|
||||
# 防止 section 粒度较粗时完全不相关的表格被无条件保护
|
||||
if min_score > 0:
|
||||
text_contexts = [c for c in text_contexts if c.get('score', 0) >= min_score]
|
||||
_table_floor = min_score * 0.3 # table 保护最低分数下限
|
||||
# 先找出所有通过阈值的 section
|
||||
passing_sections = set()
|
||||
for c in text_contexts:
|
||||
if c.get('score', 0) >= min_score:
|
||||
meta = c.get('meta', {})
|
||||
section_key = (meta.get('source', ''), meta.get('section', '') or meta.get('section_path', ''))
|
||||
if section_key[1]: # 有 section 信息的才保护
|
||||
passing_sections.add(section_key)
|
||||
|
||||
text_contexts = [
|
||||
c for c in text_contexts
|
||||
if c.get('score', 0) >= min_score
|
||||
or (
|
||||
c.get('meta', {}).get('chunk_type') == 'table'
|
||||
and c.get('score', 0) >= _table_floor
|
||||
and (c.get('meta', {}).get('source', ''), c.get('meta', {}).get('section', '') or c.get('meta', {}).get('section_path', '')) in passing_sections
|
||||
)
|
||||
]
|
||||
chart_contexts = [c for c in chart_contexts if c.get('score', 0) >= min_score]
|
||||
|
||||
# 合并:文本切片优先,图表切片补充
|
||||
@@ -258,7 +359,11 @@ def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks
|
||||
combined_contexts = text_contexts + chart_contexts[:max_chart_contexts]
|
||||
|
||||
if not _is_enum_query(query):
|
||||
return combined_contexts[:max_chunks]
|
||||
# 表格不受 max_chunks 限制(CrossEncoder 对表格评分偏低,
|
||||
# 但表格是结构化关键内容,不应因分数低而被截断)
|
||||
table_ctx = [c for c in combined_contexts if c.get('meta', {}).get('chunk_type') == 'table']
|
||||
non_table_ctx = [c for c in combined_contexts if c.get('meta', {}).get('chunk_type') != 'table']
|
||||
return non_table_ctx[:max_chunks] + table_ctx
|
||||
|
||||
def sort_key(ctx):
|
||||
meta = ctx.get('meta', {})
|
||||
@@ -312,6 +417,153 @@ def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks
|
||||
return ordered
|
||||
|
||||
|
||||
def _process_table_doc(doc: str, meta: Dict) -> str:
|
||||
"""
|
||||
处理单个切片的 doc:精简表格语义前缀 + 注入嵌入图片 URL
|
||||
|
||||
集中处理两处共用逻辑(正常路径和预算截断路径),避免重复代码。
|
||||
|
||||
Args:
|
||||
doc: 切片原始 doc
|
||||
meta: 切片 metadata
|
||||
|
||||
Returns:
|
||||
处理后的 doc
|
||||
"""
|
||||
doc = _strip_semantic_prefix(doc, meta.get('chunk_type', ''))
|
||||
if meta.get('chunk_type') == 'table' and meta.get('images_json'):
|
||||
try:
|
||||
img_list = json.loads(meta['images_json'])
|
||||
if img_list:
|
||||
img_urls = [
|
||||
f"/images/{img.get('id', '')}"
|
||||
for img in img_list
|
||||
if isinstance(img, dict) and img.get('id')
|
||||
]
|
||||
if img_urls:
|
||||
doc += "\n\n[该表格包含以下图片,可在回答中引用]: " + ", ".join(img_urls)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return doc
|
||||
|
||||
|
||||
def _section_similarity(section_a: str, section_b: str) -> float:
|
||||
"""
|
||||
计算两个章节路径的层级相似度(数据驱动,无需硬编码格式假设)。
|
||||
|
||||
策略:
|
||||
1. 如果两个路径都有数值编号(如 "2.3"),优先用精确数值匹配
|
||||
2. 否则按 " > " 层级拆分,计算 Jaccard 相似度
|
||||
3. 无数值编号时优雅降级,适用于 "第七章 附则" 或无结构文档
|
||||
|
||||
Returns: 0.0 ~ 1.0
|
||||
"""
|
||||
import re as _re
|
||||
|
||||
if not section_a or not section_b:
|
||||
return 0.0
|
||||
|
||||
# 快速路径:完全相同
|
||||
if section_a == section_b:
|
||||
return 1.0
|
||||
|
||||
# 优先精确匹配:数值编号(如 "2.3")
|
||||
num_a = _re.search(r'(\d+\.\d+)', section_a)
|
||||
num_b = _re.search(r'(\d+\.\d+)', section_b)
|
||||
if num_a and num_b:
|
||||
return 1.0 if num_a.group(1) == num_b.group(1) else 0.0
|
||||
|
||||
# 层级文本匹配:拆分路径为各级标题,计算 Jaccard 系数
|
||||
def _split_levels(path):
|
||||
parts = [p.strip() for p in path.split('>') if p.strip()]
|
||||
# 去掉常见序号前缀(如 "1.1 "、"第1章 "),保留语义部分
|
||||
cleaned = set()
|
||||
for p in parts:
|
||||
cleaned.add(_re.sub(r'^(?:\d+[\.\d]*\s*|第\s*\d+\s*章\s*|[一二三四五六七八九十]+、\s*)', '', p).strip())
|
||||
return cleaned
|
||||
|
||||
levels_a = _split_levels(section_a)
|
||||
levels_b = _split_levels(section_b)
|
||||
|
||||
if not levels_a or not levels_b:
|
||||
return 0.0
|
||||
|
||||
overlap = len(levels_a & levels_b)
|
||||
union = len(levels_a | levels_b)
|
||||
return overlap / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def _rescue_table_chunks(contexts: List[Dict], context_text: str,
|
||||
retrieval_query: str, max_rescue_chars: int = 3000) -> str:
|
||||
"""
|
||||
表格救援:当查询涉及表格但上下文预算截断了表格内容时,将最相关的表格补回。
|
||||
|
||||
根因:CrossEncoder 对 Markdown 表格格式的评分普遍偏低,
|
||||
导致 _build_context_with_budget 按分数排序时表格被排到最后并被截断。
|
||||
此函数作为安全网,确保与查询最相关的表格始终出现在上下文中。
|
||||
|
||||
Args:
|
||||
contexts: 经过 _order_text_contexts_for_prompt 处理的全部文本切片
|
||||
context_text: _build_context_with_budget 的输出
|
||||
retrieval_query: 改写后的检索查询
|
||||
max_rescue_chars: 救援表格的最大字符数
|
||||
|
||||
Returns:
|
||||
可能追加了表格内容的上下文文本
|
||||
"""
|
||||
# 1. 数据驱动检测:检索结果中是否有表格类型切片(无需硬编码关键词)
|
||||
has_table_in_contexts = any(
|
||||
ctx.get('meta', {}).get('chunk_type') == 'table' for ctx in contexts
|
||||
)
|
||||
if not has_table_in_contexts:
|
||||
return context_text
|
||||
|
||||
# 2. 检测现有上下文是否已包含表格数据(Markdown 表格至少 2 列)
|
||||
if '|' in context_text and context_text.count('|') > 4:
|
||||
return context_text
|
||||
|
||||
# 3. 从 contexts 中找被截断的表格切片
|
||||
# 使用 _in_budget_context 标记精确判断(由 _build_context_with_budget 设置)
|
||||
table_candidates = []
|
||||
for ctx in contexts:
|
||||
if ctx.get('meta', {}).get('chunk_type') != 'table':
|
||||
continue
|
||||
# 精确判断:如果 _build_context_with_budget 已标记该 chunk 为已纳入,跳过
|
||||
if ctx.get('_in_budget_context'):
|
||||
continue
|
||||
table_candidates.append(ctx)
|
||||
|
||||
if not table_candidates:
|
||||
return context_text
|
||||
|
||||
# 4. 按分数降序取最佳表格,追加到上下文
|
||||
table_candidates.sort(key=lambda c: c.get('score', 0), reverse=True)
|
||||
|
||||
rescue_parts = []
|
||||
rescue_chars = 0
|
||||
for ctx in table_candidates[:3]: # 最多救援 3 个表格
|
||||
meta = ctx.get('meta', {})
|
||||
doc = _process_table_doc(ctx.get('doc', ''), meta)
|
||||
section = meta.get('section', '') or meta.get('section_path', '')
|
||||
|
||||
part_text = ""
|
||||
if section:
|
||||
part_text += f"━ {section} ━\n"
|
||||
part_text += doc
|
||||
|
||||
if rescue_chars + len(part_text) > max_rescue_chars:
|
||||
break
|
||||
|
||||
rescue_parts.append(part_text)
|
||||
rescue_chars += len(part_text)
|
||||
|
||||
if rescue_parts:
|
||||
separator = "\n\n--- 以下为与查询最相关的表格(CrossEncoder 评分偏低,自动补入)---\n\n"
|
||||
context_text += separator + "\n\n".join(rescue_parts)
|
||||
|
||||
return context_text
|
||||
|
||||
|
||||
def _build_context_with_budget(contexts: List[Dict], max_chars: int, soft_limit: int) -> str:
|
||||
"""
|
||||
Phase 2:按字符预算构建上下文文本。
|
||||
@@ -361,22 +613,46 @@ def _build_context_with_budget(contexts: List[Dict], max_chars: int, soft_limit:
|
||||
total_chars = 0
|
||||
for key in group_order:
|
||||
group = groups[key]
|
||||
group_text = "\n\n".join(ctx.get('doc', '') for ctx in group)
|
||||
source, section = key
|
||||
|
||||
# 超过软限制后,只接受高分组
|
||||
if total_chars > soft_limit and group_max_score(key) < 0.1:
|
||||
# 判断是否包含表格切片(表格是结构化关键内容,不受预算截断)
|
||||
has_table = any(c.get('meta', {}).get('chunk_type') == 'table' for c in group)
|
||||
|
||||
# 构建组文本,表格切片附加图片 URL 供 LLM 引用
|
||||
doc_parts = []
|
||||
for ctx in group:
|
||||
doc = _process_table_doc(ctx.get('doc', ''), ctx.get('meta', {}))
|
||||
doc_parts.append(doc)
|
||||
group_text = "\n\n".join(doc_parts)
|
||||
|
||||
# 在组首插入章节路径标题行,帮助 LLM 区分不同章节
|
||||
section_header = ''
|
||||
if section:
|
||||
section_header = f"━ {section} ━"
|
||||
group_text = section_header + "\n" + group_text
|
||||
|
||||
# 超过软限制后,只接受高分组(但表格组始终保留,不因分数低被跳过)
|
||||
if total_chars > soft_limit and group_max_score(key) < 0.1 and not has_table:
|
||||
continue
|
||||
|
||||
if total_chars + len(group_text) > max_chars:
|
||||
# 尝试逐条加入该组,直到预算满
|
||||
# 先加入章节标题(如果有)
|
||||
if section_header and total_chars + len(section_header) + 2 <= max_chars:
|
||||
parts.append(section_header)
|
||||
total_chars += len(section_header) + 2
|
||||
for ctx in group:
|
||||
doc = ctx.get('doc', '')
|
||||
doc = _process_table_doc(ctx.get('doc', ''), ctx.get('meta', {}))
|
||||
if total_chars + len(doc) + 2 > max_chars: # +2 for "\n\n"
|
||||
break
|
||||
ctx['_in_budget_context'] = True # 标记已纳入上下文
|
||||
parts.append(doc)
|
||||
total_chars += len(doc) + 2
|
||||
# 超预算后一律 break,被截断的表格由 _rescue_table_chunks 补回
|
||||
break
|
||||
|
||||
for ctx in group:
|
||||
ctx['_in_budget_context'] = True # 标记已纳入上下文
|
||||
parts.append(group_text)
|
||||
total_chars += len(group_text) + 2 # +2 for "\n\n"
|
||||
|
||||
@@ -477,6 +753,17 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
||||
if len(candidates) > 1 and (candidates[0][1] - candidates[1][1]) < 0.1:
|
||||
selected_ids.append(candidates[1][0])
|
||||
|
||||
# 按 _raw_chunk_id 去重:不同 composite_key 可能指向同一个底层 chunk
|
||||
# 避免同一 chunk 产生重复引用标记(如 [3][3])
|
||||
seen_raw_ids = set()
|
||||
deduped_ids = []
|
||||
for cid in selected_ids:
|
||||
raw_id = ctx_by_chunk[cid].get('_raw_chunk_id', cid)
|
||||
if raw_id not in seen_raw_ids:
|
||||
seen_raw_ids.add(raw_id)
|
||||
deduped_ids.append(cid)
|
||||
selected_ids = deduped_ids
|
||||
|
||||
if selected_ids:
|
||||
for cid in selected_ids:
|
||||
if cid not in cited_set:
|
||||
@@ -492,16 +779,19 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
||||
result_parts.append(para + sep)
|
||||
|
||||
# 构建引用列表(按出现顺序),使用原始 chunk_id 构建 citation
|
||||
# 按 _raw_chunk_id 去重,避免同一 chunk 产生重复引用条目
|
||||
citations = []
|
||||
seen_citation_raw_ids = set()
|
||||
for composite_key in cited_chunks_ordered:
|
||||
ctx = ctx_by_chunk.get(composite_key)
|
||||
if ctx:
|
||||
raw_id = ctx.get('_raw_chunk_id') or composite_key
|
||||
if raw_id in seen_citation_raw_ids:
|
||||
continue # 同一 chunk 已在引用列表中,跳过
|
||||
seen_citation_raw_ids.add(raw_id)
|
||||
meta = ctx.get('meta', {})
|
||||
full_content = ctx.get('doc', '')
|
||||
citation = _build_citation(meta, full_content)
|
||||
# 确保 citation 中的 chunk_id 使用原始值(不含 collection 前缀)
|
||||
raw_id = ctx.get('_raw_chunk_id')
|
||||
if raw_id:
|
||||
citation['chunk_id'] = raw_id
|
||||
citations.append(citation)
|
||||
|
||||
@@ -735,8 +1025,8 @@ def score_image_relevance(query: str, meta: Dict, doc: str = '') -> float:
|
||||
|
||||
# 3. 整体文本相似度(字符级别)
|
||||
if search_text:
|
||||
# 检查查询的核心词是否在描述中
|
||||
query_core = re.sub(r'[图表图片如图所示]', '', query) # 去掉泛词
|
||||
# 复用已过滤停用词的 query_keywords,避免字符级误删(如"表现"→"现")
|
||||
query_core = "".join(query_keywords)
|
||||
if query_core:
|
||||
overlap = len(set(query_core) & set(search_text))
|
||||
score += min(overlap * 0.2, 3.0)
|
||||
@@ -763,6 +1053,77 @@ def score_image_relevance(query: str, meta: Dict, doc: str = '') -> float:
|
||||
return score
|
||||
|
||||
|
||||
def _filter_images_by_answer(selected_images: List[Dict], answer: str) -> List[Dict]:
|
||||
"""
|
||||
后置图片过滤:根据 LLM 生成的回答内容反向筛选图片。
|
||||
|
||||
只有图片描述与回答内容有足够关键词重叠时才保留,
|
||||
确保展示的图片与回答内容一致,避免不相关图片干扰用户。
|
||||
|
||||
过滤规则:
|
||||
1. 从回答中提取 2 字及以上的中文关键词(jieba 分词)
|
||||
2. 对每张图片,检查其描述(full_description / description)中匹配了多少关键词
|
||||
3. 匹配数 >= 阈值则保留,否则丢弃
|
||||
4. 如果过滤后图片为 0,保留分数最高的 1 张(兜底)
|
||||
5. 用户明确指定图号(如图 2.1)时不过滤
|
||||
|
||||
Args:
|
||||
selected_images: select_images 返回的候选图片列表
|
||||
answer: LLM 生成的回答文本
|
||||
|
||||
Returns:
|
||||
过滤后的图片列表
|
||||
"""
|
||||
if not selected_images or len(selected_images) <= 1:
|
||||
return selected_images
|
||||
|
||||
# 如果回答中提到了具体图号,说明 LLM 认为这些图是相关的,不过滤
|
||||
import re
|
||||
if re.search(r'图\s*\d+\.?\d*', answer):
|
||||
return selected_images
|
||||
|
||||
# 从回答中提取关键词
|
||||
try:
|
||||
import jieba
|
||||
answer_keywords = set(
|
||||
w for w in jieba.lcut(answer)
|
||||
if len(w) >= 2 and re.search(r'[\u4e00-\u9fff]', w)
|
||||
)
|
||||
except ImportError:
|
||||
# jieba 不可用时回退到 bigram
|
||||
chars = re.findall(r'[\u4e00-\u9fff]', answer)
|
||||
answer_keywords = set(chars[i] + chars[i + 1] for i in range(len(chars) - 1))
|
||||
|
||||
if not answer_keywords:
|
||||
return selected_images
|
||||
|
||||
# 动态阈值:回答关键词越多,阈值越高(至少匹配 15% 或 2 个关键词,取较小值)
|
||||
threshold = max(2, min(3, round(len(answer_keywords) * 0.15)))
|
||||
|
||||
filtered = []
|
||||
for img in selected_images:
|
||||
desc = img.get('full_description', '') or img.get('description', '') or ''
|
||||
if not desc:
|
||||
# 没有描述的图片,保留(无法判断)
|
||||
filtered.append(img)
|
||||
continue
|
||||
|
||||
# 计算描述与回答的关键词重叠数
|
||||
overlap = sum(1 for kw in answer_keywords if kw in desc)
|
||||
if overlap >= threshold:
|
||||
filtered.append(img)
|
||||
|
||||
# 兜底:如果过滤后为空,保留分数最高的 1 张
|
||||
if not filtered and selected_images:
|
||||
filtered = [max(selected_images, key=lambda x: x.get('score', 0))]
|
||||
|
||||
if len(filtered) < len(selected_images):
|
||||
logger.info(f"[图片后置过滤] {len(selected_images)} → {len(filtered)} 张 "
|
||||
f"(回答关键词 {len(answer_keywords)} 个, 阈值 {threshold})")
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
"""
|
||||
选择要展示的图片(打分排序 + 预算控制)
|
||||
@@ -792,26 +1153,39 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
"""
|
||||
import re
|
||||
|
||||
# 动态预算:列举型查询允许更多图片
|
||||
list_keywords = ["哪些", "有什么", "包含", "列出", "所有", "全部"]
|
||||
is_list_query = any(kw in query for kw in list_keywords)
|
||||
# 动态预算:数据驱动,不依赖硬编码关键词列表
|
||||
# 核心策略:宽松预选 + 后置过滤(_filter_images_by_answer)精准裁剪
|
||||
|
||||
# 检测查询中的图片编号(如 "图2.1")
|
||||
# 精确查图:用户指定了具体图号(如 "图2.3")—— 结构化模式匹配,非硬编码
|
||||
figure_pattern = r'图\s*(\d+\.?\d*)'
|
||||
figure_matches = re.findall(figure_pattern, query)
|
||||
has_figure_query = bool(figure_matches)
|
||||
|
||||
# 新增:从检索文本中提取图表引用(见表2.2、见图2.5 等)
|
||||
# 同时记录引用所在的文件来源
|
||||
# 数据驱动的图片意图检测:检查检索结果中是否包含图片/图表类型切片
|
||||
# 原理:如果向量检索返回了 image/chart 类型 chunk 或含 images_json 的 table chunk,
|
||||
# 说明知识库中存在与查询语义相关的图片内容,应给予展示机会
|
||||
has_image_data = False
|
||||
_image_chunk_count = 0
|
||||
_table_image_count = 0
|
||||
for ctx in contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
ct = meta.get('chunk_type', '')
|
||||
if ct in ('image', 'chart'):
|
||||
_image_chunk_count += 1
|
||||
has_image_data = True
|
||||
if ct == 'table' and meta.get('images_json'):
|
||||
_table_image_count += 1
|
||||
has_image_data = True
|
||||
|
||||
# 从检索文本中提取图表引用(见表2.2、见图2.5 等)
|
||||
# 重要:只从语义相关的 top 5 文本块提取,避免不相关引用干扰
|
||||
referenced_figures = {} # {图号: set(文件来源)}
|
||||
referenced_tables = {} # {表号: set(文件来源)}
|
||||
|
||||
for ctx in contexts[:5]: # 只从前5个最相关的文本块提取引用
|
||||
for ctx in contexts[:5]:
|
||||
doc_text = ctx.get('doc', '')
|
||||
source = ctx.get('meta', {}).get('source', '')
|
||||
|
||||
# 提取 "见图X.X"、"如图X.X" 或单独的 "图X.X"
|
||||
fig_refs = re.findall(r'(?:[见如])?图\s*(\d+\.?\d*)', doc_text)
|
||||
for fig_num in fig_refs:
|
||||
if fig_num not in referenced_figures:
|
||||
@@ -819,7 +1193,6 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
if source:
|
||||
referenced_figures[fig_num].add(source)
|
||||
|
||||
# 提取 "见表X.X"、"如表X.X" 或单独的 "表X.X"
|
||||
table_refs = re.findall(r'(?:[见如])?表\s*(\d+\.?\d*)', doc_text)
|
||||
for table_num in table_refs:
|
||||
if table_num not in referenced_tables:
|
||||
@@ -831,63 +1204,84 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
|
||||
# 获取检索结果中涉及的主要文件来源
|
||||
primary_sources = set()
|
||||
for ctx in contexts[:5]: # 只看前5个最相关的
|
||||
for ctx in contexts[:5]:
|
||||
source = ctx.get('meta', {}).get('source', '')
|
||||
if source:
|
||||
primary_sources.add(source)
|
||||
|
||||
# 图片意图检测:区分精确查图和泛指
|
||||
strong_image_keywords = ["示意图", "流程图", "结构图", "过程线", "曲线图", "分布图", "图示", "看图", "显示图"]
|
||||
weak_image_keywords = ["图片", "图表", "如图", "图", "统计"]
|
||||
|
||||
has_strong_image_intent = any(kw in query for kw in strong_image_keywords)
|
||||
has_weak_image_intent = any(kw in query for kw in weak_image_keywords)
|
||||
|
||||
# 精确查图:用户指定了具体图号(如 "图2.3"),只返回最匹配的 1-2 张
|
||||
# 动态预算:根据检索结果数据驱动设置
|
||||
# 后置过滤 _filter_images_by_answer 会根据回答内容精准裁剪
|
||||
if has_figure_query:
|
||||
# 精确查图:用户指定了具体图号
|
||||
MAX_IMAGES = 2
|
||||
MIN_SCORE = 5.0 # 精确匹配应该高分
|
||||
# 强图片意图:明确要看某种图
|
||||
elif has_strong_image_intent:
|
||||
MAX_IMAGES = 3
|
||||
MIN_SCORE = 5.0 # 提高阈值,避免不相关图片通过
|
||||
# 列举型查询
|
||||
elif is_list_query:
|
||||
MIN_SCORE = 5.0
|
||||
elif has_image_data:
|
||||
# 检索结果中有图片数据 → 宽松预算,给后置过滤留足候选空间
|
||||
MAX_IMAGES = 5
|
||||
MIN_SCORE = 3.0
|
||||
# 有图表引用:检索文本中提到了图表
|
||||
MIN_SCORE = 2.0
|
||||
elif has_referenced_figures:
|
||||
# 检索文本中引用了图表编号
|
||||
MAX_IMAGES = 3
|
||||
MIN_SCORE = 2.0 # 降低阈值,让引用的图表能通过
|
||||
# 弱图片意图:只是提到"图"字,可能是泛指(如 "发电量图")
|
||||
elif has_weak_image_intent:
|
||||
MAX_IMAGES = 1 # 只返回最相关的一张
|
||||
MIN_SCORE = 2.0 # 降低阈值,让语义相关的图片能通过
|
||||
# 普通查询
|
||||
MIN_SCORE = 2.0
|
||||
else:
|
||||
# 无图片数据 → 保守默认值
|
||||
MAX_IMAGES = 2
|
||||
MIN_SCORE = 3.0
|
||||
|
||||
# 获取检索结果中涉及的主要章节(只看前 3 个最相关的文本块)
|
||||
primary_sections = set()
|
||||
# 动态调整:当表格嵌入大量图片时,提升上限以展示完整内容
|
||||
if has_image_data and _table_image_count > 0:
|
||||
total_table_images = 0
|
||||
for ctx in contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
if meta.get('chunk_type') == 'table' and meta.get('images_json'):
|
||||
try:
|
||||
total_table_images += len(json.loads(meta['images_json']))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
if total_table_images > MAX_IMAGES:
|
||||
MAX_IMAGES = min(total_table_images, 15) # 上限 15,防止图片过多
|
||||
|
||||
# 获取检索结果中涉及的主要章节路径(只看前 3 个最相关的文本块)
|
||||
primary_section_paths = set()
|
||||
for ctx in contexts[:3]:
|
||||
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||
if section:
|
||||
# 提取章节编号
|
||||
# 优先匹配 X.X 格式(如 "2.3发电"),再匹配 第X章 格式
|
||||
section_num = re.search(r'(\d+\.\d+)', section)
|
||||
if not section_num:
|
||||
# 尝试匹配 "第X章" 格式
|
||||
chapter_match = re.search(r'第\s*(\d+)\s*章', section)
|
||||
if chapter_match:
|
||||
section_num = chapter_match
|
||||
if section_num:
|
||||
primary_sections.add(section_num.group(1))
|
||||
primary_section_paths.add(section)
|
||||
|
||||
# ========== P1.5 预计算:表格主题相关性评分 ==========
|
||||
# 从查询中提取关键词片段(使用 jieba 分词 + 2字及以上的词,自动过滤停用词)
|
||||
try:
|
||||
import jieba
|
||||
_query_kw_segments = [w for w in jieba.lcut(query)
|
||||
if len(w) >= 2 and re.search(r'[\u4e00-\u9fff]', w)]
|
||||
except ImportError:
|
||||
# jieba 不可用时回退到 bigram
|
||||
_chars = re.findall(r'[\u4e00-\u9fff]', query)
|
||||
_query_kw_segments = [_chars[i] + _chars[i+1] for i in range(len(_chars) - 1)]
|
||||
|
||||
# 对所有含 images_json 的表格切片计算主题匹配分
|
||||
_table_topic_scores = {}
|
||||
for _tc in contexts:
|
||||
_tm = _tc.get('meta', {})
|
||||
if _tm.get('chunk_type') == 'table' and _tm.get('images_json'):
|
||||
_t_section = (_tm.get('section', '') or _tm.get('section_path', ''))
|
||||
_t_title = _tm.get('title', '') or ''
|
||||
_t_combined = _t_title + _t_section
|
||||
_score = sum(1 for kw in _query_kw_segments if kw in _t_combined)
|
||||
_table_topic_scores[id(_tc)] = _score
|
||||
|
||||
# 自适应阈值:要求至少匹配 70% 的最佳表格得分(至少 2 分)
|
||||
# 例如查询 "设施设备的参考样式" → 4 个关键词 → 最佳匹配 4 → 阈值 max(2, 2) = 2
|
||||
# "形象识别标识" 只匹配 "参考"+"样式" = 2 → 但阈值=3(70%of4)时被过滤
|
||||
_best_table_score = max(_table_topic_scores.values()) if _table_topic_scores else 0
|
||||
_table_topic_threshold = max(2, round(_best_table_score * 0.7)) if _best_table_score >= 2 else 0
|
||||
|
||||
scored_images = []
|
||||
for ctx in contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
chunk_type = meta.get('chunk_type', 'text')
|
||||
s = None # P1 评分,用于 P1.5 继承
|
||||
doc = ctx.get('doc', '') # 默认文档内容
|
||||
|
||||
# 处理图片类型和有关联图片的表格类型
|
||||
if meta.get('image_path') and chunk_type in ('image', 'chart', 'table'):
|
||||
@@ -917,16 +1311,19 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
|
||||
# 图片章节
|
||||
img_section = meta.get('section', '') or meta.get('section_path', '')
|
||||
# 只匹配 X.X 格式的章节号,避免匹配年份
|
||||
img_section_num = re.search(r'(\d+\.\d+)', img_section)
|
||||
img_section_id = img_section_num.group(1) if img_section_num else None
|
||||
|
||||
# ========== 核心修复:图片必须与主要文本切片章节关联 ==========
|
||||
# 如果有主要章节,且图片章节不在其中,大幅降低分数
|
||||
# ========== 章节关联检测:基于层级相似度,无需硬编码格式假设 ==========
|
||||
# 计算图片章节与主要检索结果的最高相似度
|
||||
section_penalty = 0.0
|
||||
if primary_sections and img_section_id and img_section_id not in primary_sections:
|
||||
max_section_sim = 0.0
|
||||
if primary_section_paths:
|
||||
for ps in primary_section_paths:
|
||||
sim = _section_similarity(img_section, ps)
|
||||
max_section_sim = max(max_section_sim, sim)
|
||||
# 当相似度低于阈值且有足够的章节信息时,判定为不相关
|
||||
if primary_section_paths and img_section and max_section_sim < 0.3:
|
||||
# 图片章节与主要检索结果不匹配,惩罚
|
||||
section_penalty = -5.0 # 大幅降低分数
|
||||
section_penalty = -5.0
|
||||
# 除非图片被文本切片明确引用
|
||||
is_referenced = False
|
||||
for fig_num in referenced_figures:
|
||||
@@ -949,10 +1346,10 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
for fig_num, sources in referenced_figures.items():
|
||||
# 只检查 doc 字段,不检查 meta(避免 section 中的误匹配)
|
||||
if f"图{fig_num}" in doc or f"图 {fig_num}" in doc:
|
||||
# 检查图片章节是否与主要章节匹配
|
||||
section_match = img_section_id and img_section_id in primary_sections
|
||||
# 使用层级相似度判断章节关联性
|
||||
section_match = max_section_sim >= 0.3
|
||||
|
||||
# P2:只有章节匹配才加分,移除"s >= 5.0"漏洞
|
||||
# 章节匹配时才加分
|
||||
if section_match:
|
||||
# 图号匹配加分
|
||||
s += 8.0
|
||||
@@ -967,10 +1364,10 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
for table_num, sources in referenced_tables.items():
|
||||
# 只检查 doc 字段
|
||||
if f"表{table_num}" in doc or f"表 {table_num}" in doc:
|
||||
# 检查图片章节是否与主要章节匹配
|
||||
section_match = img_section_id and img_section_id in primary_sections
|
||||
# 使用层级相似度判断章节关联性
|
||||
section_match = max_section_sim >= 0.3
|
||||
|
||||
# P2:只有章节匹配才加分,移除"s >= 5.0"漏洞
|
||||
# 章节匹配时才加分
|
||||
if section_match:
|
||||
# 表号匹配加分
|
||||
s += 8.0
|
||||
@@ -998,6 +1395,47 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
# ========== P1.5:处理表格切片的 images_json(跨页表格多图)==========
|
||||
# 当表格切片有 images_json 字段时,添加所有关联图片
|
||||
if chunk_type == 'table' and meta.get('images_json'):
|
||||
# 如果 P1 未执行(表格无 image_path),需要独立计算分数
|
||||
if s is None:
|
||||
doc = ctx.get('image_description', '') or meta.get('vlm_desc', '') or ctx.get('doc', '')
|
||||
s = score_image_relevance(query, meta, doc)
|
||||
|
||||
# 章节相关性过滤:使用层级相似度,无需硬编码章节格式假设
|
||||
table_section = meta.get('section', '') or meta.get('section_path', '')
|
||||
|
||||
section_relevant = True
|
||||
if primary_section_paths and table_section:
|
||||
# 计算表格章节与所有主要章节的最高相似度
|
||||
max_sim = max(
|
||||
(_section_similarity(table_section, ps) for ps in primary_section_paths),
|
||||
default=0.0
|
||||
)
|
||||
if max_sim < 0.3:
|
||||
section_relevant = False
|
||||
elif primary_section_paths and not table_section:
|
||||
# 表格无章节信息时优雅降级:不做章节过滤,仅依赖主题分数
|
||||
pass
|
||||
|
||||
if not section_relevant:
|
||||
# 例外:如果表格标题/内容被查询直接提及,仍视为相关
|
||||
table_title = meta.get('title', '') or ''
|
||||
table_doc = ctx.get('doc', '') or ''
|
||||
if table_title and table_title in query:
|
||||
section_relevant = True
|
||||
elif table_doc and any(kw in table_doc for kw in query.split() if len(kw) >= 2):
|
||||
section_relevant = True
|
||||
|
||||
if not section_relevant:
|
||||
logger.debug(f"P1.5 跳过无关表格图片: path={table_section}, title={meta.get('title', '')}")
|
||||
continue # 跳过此表格的所有嵌入图片
|
||||
|
||||
# 标题/主题相关性过滤:基于预计算的关键词匹配评分
|
||||
# 如果最佳表格得分 >= 2,则过滤掉得分为 0 的表格
|
||||
_this_topic_score = _table_topic_scores.get(id(ctx), 0)
|
||||
if _this_topic_score < _table_topic_threshold:
|
||||
logger.debug(f"P1.5 跳过主题不匹配表格: topic_score={_this_topic_score}, threshold={_table_topic_threshold}, title={meta.get('title', '')}")
|
||||
continue
|
||||
|
||||
try:
|
||||
images_list = json.loads(meta['images_json'])
|
||||
for img_info in images_list:
|
||||
@@ -1354,6 +1792,18 @@ def rag():
|
||||
except Exception as e:
|
||||
logger.debug(f"创建会话失败: {e}")
|
||||
|
||||
# ==================== collections 历史推断(开发环境) ====================
|
||||
# 当 collections 未显式指定(或仅为默认 public_kb)且有会话历史时,
|
||||
# 从历史消息中推断上次使用的 KB,自动恢复以避免用户忘切 KB
|
||||
if (not collections or collections == ['public_kb']) and history:
|
||||
for _msg in reversed(history):
|
||||
if _msg.get("role") == "assistant":
|
||||
_meta = _msg.get("metadata", {})
|
||||
if isinstance(_meta, dict) and _meta.get("collections"):
|
||||
collections = _meta["collections"]
|
||||
logger.info(f"[KB推断] 从历史推断 collections: {collections}")
|
||||
break
|
||||
|
||||
# 提前获取 session_repo 引用,避免在生成器内部访问 current_app
|
||||
# (生成器执行时应用上下文可能已结束)
|
||||
session_repo_ref = None
|
||||
@@ -1453,6 +1903,9 @@ def rag():
|
||||
except Exception as e:
|
||||
logger.warning(f"意图分析失败: {e},继续执行检索流程")
|
||||
|
||||
# 构建检索查询:使用改写后的完整问题(解决追问偏离问题)
|
||||
retrieval_query = intent.rewritten_query if (intent and intent.rewritten_query) else message
|
||||
|
||||
# 1.5 语义缓存检查(跳过检索+生成全流程)
|
||||
_semantic_cache_emb = None
|
||||
try:
|
||||
@@ -1461,9 +1914,13 @@ def rag():
|
||||
_sc = get_semantic_cache()
|
||||
_eng = _get_eng()
|
||||
if _sc and _eng and hasattr(_eng, 'embedding_model'):
|
||||
_semantic_cache_emb = _eng.embedding_model.encode(message)
|
||||
# 缓存 key 使用 retrieval_query + collections,避免不同上下文的追问命中错误缓存
|
||||
_cache_collections = ','.join(sorted(collections)) if collections else ''
|
||||
_cache_key_text = f"{retrieval_query}|{_cache_collections}"
|
||||
_semantic_cache_emb = _eng.embedding_model.encode(_cache_key_text)
|
||||
cached = _sc.get(_semantic_cache_emb)
|
||||
if cached is not None:
|
||||
# 防御性校验:必须是 RAG 回答缓存(非 intent_analyzer 缓存),且回答非空
|
||||
if cached is not None and cached.get("cache_type") == "rag_answer" and cached.get("answer"):
|
||||
logger.info(f"[语义缓存] 命中: {message[:50]}...")
|
||||
cached_answer = cached.get("answer", "")
|
||||
# 流式返回缓存的答案
|
||||
@@ -1504,7 +1961,7 @@ def rag():
|
||||
sub_queries = intent.sub_queries
|
||||
|
||||
search_result = search_hybrid(
|
||||
message,
|
||||
retrieval_query,
|
||||
top_k=RAG_SEARCH_TOP_K,
|
||||
candidates=RAG_SEARCH_CANDIDATES,
|
||||
allowed_collections=collections,
|
||||
@@ -1525,18 +1982,19 @@ def rag():
|
||||
metas = search_result.get('metadatas', [[]])[0]
|
||||
scores = search_result.get('scores', [[]])[0]
|
||||
|
||||
# 图片相关性提升:检测图片编号或强意图
|
||||
# 图片相关性提升:数据驱动检测(无硬编码关键词)
|
||||
import re
|
||||
figure_pattern = r'图\s*(\d+\.?\d*)'
|
||||
figure_matches = re.findall(figure_pattern, message)
|
||||
figure_matches = re.findall(figure_pattern, retrieval_query)
|
||||
has_figure_query = bool(figure_matches)
|
||||
|
||||
# Bug 4 修复:缩小 strong_image_keywords,移除歧义词
|
||||
# "过程线" 是水文术语不是图片意图,"图片"/"图表"/"如图" 太泛
|
||||
strong_image_keywords = ["示意图", "流程图", "结构图", "曲线图", "分布图", "图示", "看图", "显示图", "给我看"]
|
||||
has_image_intent = has_figure_query or any(kw in message for kw in strong_image_keywords)
|
||||
# 数据驱动:检查检索结果中是否有图片/图表类型切片
|
||||
has_image_data = any(
|
||||
m.get('chunk_type') in ('image', 'chart') for m in metas
|
||||
)
|
||||
has_image_intent = has_figure_query or has_image_data
|
||||
|
||||
# Bug 2 修复:不重排 contexts,只在 meta 里打标记给后续的 select_images 用
|
||||
# 给图片/图表切片打 boost 标记,供后续 select_images 使用
|
||||
if has_image_intent:
|
||||
for i, (doc, meta, score) in enumerate(zip(docs, metas, scores)):
|
||||
if meta.get('chunk_type') in ('image', 'chart'):
|
||||
@@ -1658,18 +2116,12 @@ def rag():
|
||||
missing_figures = referenced_figures - existing_figure_images
|
||||
missing_tables = referenced_tables - existing_table_images
|
||||
|
||||
# 计算主要章节(用于补充检索过滤)
|
||||
primary_sections_for_supplement = set()
|
||||
# 计算主要章节路径(用于补充检索过滤)
|
||||
primary_section_paths_for_supp = set()
|
||||
for ctx in text_contexts[:3]:
|
||||
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||
if section:
|
||||
section_num = re.search(r'(\d+\.\d+)', section)
|
||||
if not section_num:
|
||||
chapter_match = re.search(r'第\s*(\d+)\s*章', section)
|
||||
if chapter_match:
|
||||
section_num = chapter_match
|
||||
if section_num:
|
||||
primary_sections_for_supplement.add(section_num.group(1))
|
||||
primary_section_paths_for_supp.add(section)
|
||||
|
||||
if missing_figures or missing_tables:
|
||||
# 补充检索
|
||||
@@ -1726,14 +2178,18 @@ def rag():
|
||||
|
||||
if is_match:
|
||||
# 额外检查:图片章节是否与主要章节匹配
|
||||
# 避免补充检索到不相关的图片
|
||||
# 使用层级相似度判断,无需硬编码格式假设
|
||||
supp_section = supp_meta.get('section', '') or supp_meta.get('section_path', '')
|
||||
supp_section_num = re.search(r'(\d+\.\d+)', supp_section)
|
||||
supp_section_id = supp_section_num.group(1) if supp_section_num else None
|
||||
|
||||
# 如果图片章节不在主要章节中,跳过
|
||||
if primary_sections_for_supplement and supp_section_id and supp_section_id not in primary_sections_for_supplement:
|
||||
# 不是主要章节的图片,跳过
|
||||
if primary_section_paths_for_supp and supp_section:
|
||||
supp_max_sim = max(
|
||||
(_section_similarity(supp_section, ps) for ps in primary_section_paths_for_supp),
|
||||
default=0.0
|
||||
)
|
||||
if supp_max_sim < 0.3:
|
||||
continue
|
||||
elif primary_section_paths_for_supp and not supp_section:
|
||||
# 补充检索的图片无章节信息时优雅降级:跳过
|
||||
continue
|
||||
|
||||
# Bug 6a 修复:补充检索的图片也要做 full_description 替换
|
||||
@@ -1772,12 +2228,12 @@ def rag():
|
||||
import asyncio
|
||||
from knowledge.lazy_enhance import enhance_retrieved_chunks
|
||||
kb_name = collections[0] if collections else 'public_kb'
|
||||
asyncio.run(enhance_retrieved_chunks(contexts, message, kb_name))
|
||||
asyncio.run(enhance_retrieved_chunks(contexts, retrieval_query, kb_name))
|
||||
except Exception as e:
|
||||
logger.warning(f"懒加载增强失败: {e}")
|
||||
|
||||
# 3. 选择要展示的图片(Phase 5)
|
||||
selected_images = select_images(contexts, message)
|
||||
selected_images = select_images(contexts, retrieval_query)
|
||||
|
||||
# 调试事件:图片选择详情
|
||||
if IS_DEV:
|
||||
@@ -1785,16 +2241,32 @@ def rag():
|
||||
|
||||
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
||||
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
||||
text_contexts = _order_text_contexts_for_prompt(contexts, message, MAX_CONTEXT_CHUNKS,
|
||||
text_contexts = _order_text_contexts_for_prompt(contexts, retrieval_query, MAX_CONTEXT_CHUNKS,
|
||||
min_score=RERANK_CONTEXT_MIN_SCORE)
|
||||
# Phase 2:按字符预算构建上下文
|
||||
_is_comparison = intent and intent.intent == "comparison"
|
||||
if _is_enum_query(message) or _is_comparison:
|
||||
if _is_enum_query(retrieval_query) or _is_comparison:
|
||||
# 列举类 / 对比类查询:保持原始顺序,不做预算截断
|
||||
context_text = "\n\n".join([ctx.get('doc', '') for ctx in text_contexts])
|
||||
# 仍然注入 section_path 标题,帮助 LLM 区分不同章节
|
||||
enum_parts = []
|
||||
prev_section = None
|
||||
for ctx in text_contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
section = meta.get('section', '') or meta.get('section_path', '')
|
||||
if section and section != prev_section:
|
||||
enum_parts.append(f"━ {section} ━")
|
||||
prev_section = section
|
||||
# 精简表格切片:去除冗余的语义增强前缀
|
||||
doc = _strip_semantic_prefix(ctx.get('doc', ''), meta.get('chunk_type', ''))
|
||||
enum_parts.append(doc)
|
||||
context_text = "\n\n".join(enum_parts)
|
||||
else:
|
||||
context_text = _build_context_with_budget(text_contexts, CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT)
|
||||
|
||||
# 表格救援:CrossEncoder 对表格评分偏低,导致表格被预算截断
|
||||
# 当查询涉及表格但上下文中没有表格数据时,从被截断的切片中补回
|
||||
context_text = _rescue_table_chunks(text_contexts, context_text, retrieval_query)
|
||||
|
||||
# Phase 4:计算置信度分数(top-3 平均 Rerank 分数)
|
||||
_top_scores = [ctx.get('score', 0) for ctx in text_contexts[:3]]
|
||||
_confidence_score = round(sum(_top_scores) / len(_top_scores), 4) if _top_scores else 0.0
|
||||
@@ -1802,6 +2274,17 @@ def rag():
|
||||
# Bug 6b 优化:直接使用 selected_images 中的 full_description
|
||||
# 这样 LLM 既能看到文本切片,也能知道图片内容
|
||||
if selected_images:
|
||||
# 区分表格嵌入图片和独立图片
|
||||
has_table_embedded_images = any(
|
||||
img.get('type') == 'table_image' for img in selected_images
|
||||
)
|
||||
# 检查上下文中是否有表格含嵌入图片
|
||||
has_table_with_images = any(
|
||||
ctx.get('meta', {}).get('chunk_type') == 'table'
|
||||
and ctx.get('meta', {}).get('images_json')
|
||||
for ctx in text_contexts
|
||||
)
|
||||
|
||||
image_descriptions = []
|
||||
for i, img in enumerate(selected_images, 1):
|
||||
# 直接使用 select_images 时带上的 full_description
|
||||
@@ -1814,6 +2297,16 @@ def rag():
|
||||
image_descriptions.append(f"【图片{i}】{full_desc}{source_info}")
|
||||
if image_descriptions:
|
||||
context_text += "\n\n【相关图片信息】\n" + "\n\n".join(image_descriptions)
|
||||
|
||||
# 根据图片类型给出不同的回答指令
|
||||
if has_table_embedded_images and has_table_with_images:
|
||||
context_text += (
|
||||
"\n\n【回答要求】参考资料中包含表格及其嵌入图片。"
|
||||
"请以**表格形式**呈现数据(保持原始表格结构),"
|
||||
"并在对应单元格中使用 `` 格式嵌入图片。"
|
||||
"不要将表格内容转为纯文本描述,不要把图片与表格分开展示。"
|
||||
)
|
||||
else:
|
||||
# 添加指令让 LLM 介绍图片
|
||||
context_text += "\n\n【回答要求】回答时请简要介绍每张图片的内容和用途。"
|
||||
|
||||
@@ -1847,7 +2340,7 @@ def rag():
|
||||
)
|
||||
enhanced_context = instruction_instruction + "\n\n" + enhanced_context
|
||||
|
||||
if _is_enum_query(message):
|
||||
if _is_enum_query(retrieval_query):
|
||||
enum_instruction = (
|
||||
"\n\n【回答要求】如果参考资料中包含编号列表、禁止情形、要求或条款,"
|
||||
"请按资料中的原始顺序完整列出;不要合并相邻条目,不要跳项,"
|
||||
@@ -1914,6 +2407,9 @@ def rag():
|
||||
# else: 没有匹配到,保留原选择(不再截断到1张)
|
||||
# else: LLM 没有提图号,保留原选择(不再截断到1张)
|
||||
|
||||
# 后置图片过滤:用回答内容反向筛选图片,确保图片与回答一致
|
||||
selected_images = _filter_images_by_answer(selected_images, full_answer_text)
|
||||
|
||||
rich_media = {'images': selected_images, 'tables': [], 'sections': []}
|
||||
|
||||
# 7. 去掉 LLM 添加的数字引用标记,避免与后端引用重复
|
||||
@@ -1941,6 +2437,8 @@ def rag():
|
||||
assistant_metadata['sources'] = sources
|
||||
if citation_result.get('citations'):
|
||||
assistant_metadata['citations'] = citation_result['citations']
|
||||
# 记录本次检索使用的向量库(用于后续追问时自动恢复 KB 选择)
|
||||
assistant_metadata['collections'] = collections
|
||||
session_repo_ref.add_message(session_id, 'assistant', filtered_answer, assistant_metadata)
|
||||
# 更新会话最后活跃时间
|
||||
if hasattr(session_repo_ref, 'update_last_active'):
|
||||
@@ -1990,9 +2488,12 @@ def rag():
|
||||
from core.engine import get_engine as _get_eng
|
||||
_eng = _get_eng()
|
||||
if _eng and hasattr(_eng, 'embedding_model'):
|
||||
_semantic_cache_emb = _eng.embedding_model.encode(message)
|
||||
_cache_collections = ','.join(sorted(collections)) if collections else ''
|
||||
_cache_key_text = f"{retrieval_query}|{_cache_collections}"
|
||||
_semantic_cache_emb = _eng.embedding_model.encode(_cache_key_text)
|
||||
if _semantic_cache_emb is not None:
|
||||
_sc.set(_semantic_cache_emb, {
|
||||
"cache_type": "rag_answer",
|
||||
"answer": filtered_answer,
|
||||
"sources": sources,
|
||||
"citations": citation_result.get("citations", []),
|
||||
|
||||
@@ -45,6 +45,7 @@ import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
from werkzeug.utils import secure_filename
|
||||
from auth.gateway import require_gateway_auth
|
||||
from config import DEV_MODE
|
||||
from core.status_codes import (
|
||||
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST, SUCCESS,
|
||||
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION, NOT_FOUND,
|
||||
@@ -171,9 +172,9 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
|
||||
文件内容或错误响应
|
||||
|
||||
Note:
|
||||
仅在 DEV_MODE=true 时可用
|
||||
仅在 DEV_MODE=true 时可用(需在 .env 中显式设置)
|
||||
"""
|
||||
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||||
if not DEV_MODE:
|
||||
return error_response("FORBIDDEN", FORBIDDEN, "仅开发环境可用", http_status=403)
|
||||
|
||||
from config import DOCUMENTS_PATH
|
||||
|
||||
@@ -71,7 +71,7 @@ def get_history(session_id):
|
||||
user_id = request.current_user["user_id"]
|
||||
|
||||
# 验证会话归属
|
||||
sessions = session_manager.get_user_sessions(user_id)
|
||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
||||
session_ids = [s["session_id"] for s in sessions]
|
||||
|
||||
if session_id not in session_ids:
|
||||
@@ -92,7 +92,7 @@ def delete_session(session_id):
|
||||
user_id = request.current_user["user_id"]
|
||||
|
||||
# 验证会话归属
|
||||
sessions = session_manager.get_user_sessions(user_id)
|
||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
||||
session_ids = [s["session_id"] for s in sessions]
|
||||
|
||||
if session_id not in session_ids:
|
||||
@@ -113,7 +113,7 @@ def clear_history(session_id):
|
||||
user_id = request.current_user["user_id"]
|
||||
|
||||
# 验证会话归属
|
||||
sessions = session_manager.get_user_sessions(user_id)
|
||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
||||
session_ids = [s["session_id"] for s in sessions]
|
||||
|
||||
if session_id not in session_ids:
|
||||
|
||||
@@ -20,12 +20,12 @@
|
||||
|
||||
## 模式说明
|
||||
|
||||
开发模式 (DEV_MODE=true,默认):
|
||||
开发模式 (DEV_MODE=true):
|
||||
- 支持 mock token 模拟用户:Authorization: Bearer mock-token-admin
|
||||
- 无 Header 时自动使用开发测试用户
|
||||
- 适用于前端测试和开发调试
|
||||
|
||||
生产模式 (DEV_MODE=false):
|
||||
生产模式 (DEV_MODE=false,默认):
|
||||
- 不需要 Header,直接放行
|
||||
- 权限由后端完全控制,通过 collections 参数传入
|
||||
- RAG 服务完全无状态,只负责问答检索
|
||||
@@ -34,17 +34,11 @@
|
||||
from functools import wraps
|
||||
from flask import request, jsonify
|
||||
from typing import Dict, Optional
|
||||
import os
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 加载 .env 文件(从项目根目录)
|
||||
env_path = Path(__file__).parent.parent / '.env'
|
||||
load_dotenv(env_path)
|
||||
from config import DEV_MODE
|
||||
|
||||
|
||||
# ==================== 模拟用户数据(开发环境)====================
|
||||
# 用于前端模拟登录测试,仅 DEV_MODE=true 时生效
|
||||
# 用于前端模拟登录测试,仅 DEV_MODE=true 时生效(需在 .env 中显式开启)
|
||||
MOCK_USERS = {
|
||||
'admin': {
|
||||
'user_id': 'admin001',
|
||||
@@ -83,19 +77,19 @@ def require_gateway_auth(f):
|
||||
"""
|
||||
网关认证装饰器 - 从 Header 读取用户信息
|
||||
|
||||
开发模式 (DEV_MODE=true,默认):
|
||||
开发模式 (DEV_MODE=true):
|
||||
- 支持 mock token: Authorization: Bearer mock-token-admin
|
||||
- 无 Header 时自动使用开发测试用户(admin 角色)
|
||||
|
||||
生产模式 (DEV_MODE=false):
|
||||
生产模式 (DEV_MODE=false,默认):
|
||||
- 不需要 Header,直接放行
|
||||
- 用户信息设为默认值
|
||||
- 权限由后端通过 collections 参数控制
|
||||
"""
|
||||
@wraps(f)
|
||||
def decorated(*args, **kwargs):
|
||||
# 开发模式开关(默认开启,生产环境设置 DEV_MODE=false)
|
||||
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||
# 开发模式开关(统一由 config.py 管理)
|
||||
dev_mode = DEV_MODE
|
||||
|
||||
# 开发模式:支持 mock token
|
||||
if dev_mode:
|
||||
@@ -202,11 +196,24 @@ def can_delete_collection(role: str) -> bool:
|
||||
|
||||
def require_role(*roles):
|
||||
"""
|
||||
兼容旧代码 - 权限由后端管理,此装饰器不再执行权限验证
|
||||
角色验证装饰器(开发和生产环境均生效)
|
||||
|
||||
需搭配 @require_gateway_auth 使用(先设置 current_user,再验证角色)。
|
||||
|
||||
开发模式: 检查 mock token 对应用户的角色
|
||||
生产模式: 检查网关注入的 X-User-Role Header
|
||||
"""
|
||||
def decorator(f):
|
||||
@wraps(f)
|
||||
def decorated(*args, **kwargs):
|
||||
if roles:
|
||||
user = get_current_user()
|
||||
if user is None or user.get('role') not in roles:
|
||||
from flask import jsonify
|
||||
return jsonify({
|
||||
"error": "权限不足,需要角色: {}".format(', '.join(roles)),
|
||||
"status": "FORBIDDEN"
|
||||
}), 403
|
||||
return f(*args, **kwargs)
|
||||
return decorated
|
||||
return decorator
|
||||
@@ -214,11 +221,12 @@ def require_role(*roles):
|
||||
|
||||
def require_collection_permission(operation: str):
|
||||
"""
|
||||
兼容旧代码 - 权限由后端管理,此装饰器不再执行权限验证
|
||||
集合权限验证装饰器(开发模式下为占位实现,生产环境权限由网关控制)
|
||||
"""
|
||||
def decorator(f):
|
||||
@wraps(f)
|
||||
def decorated(*args, **kwargs):
|
||||
# 生产环境下权限由网关/后端统一管控,此处放行
|
||||
return f(*args, **kwargs)
|
||||
return decorated
|
||||
return decorator
|
||||
|
||||
@@ -201,6 +201,18 @@ MINERU_DEVICE_MODE = os.getenv("MINERU_DEVICE_MODE", "cpu") # cpu / cuda
|
||||
MINERU_API_TOKEN = os.getenv("MINERU_API_TOKEN", "") # 在 https://mineru.net/apiManage/token 申请
|
||||
MINERU_API_URL = os.getenv("MINERU_API_URL", "https://mineru.net/api/v4/extract/task")
|
||||
MINERU_PREFER_ONLINE = os.getenv("MINERU_PREFER_ONLINE", "true").lower() == "true"
|
||||
MINERU_PREFER_V2 = os.getenv("MINERU_PREFER_V2", "true").lower() == "true" # 优先使用 v2 格式(含 style 信息)
|
||||
|
||||
# 标题识别规则引擎
|
||||
# 规则定义见 parsers/heading_rules.py,支持通过 config.py 覆盖
|
||||
HEADING_RULES_CONFIG = None # None=使用内置默认规则; dict 列表=自定义规则覆盖
|
||||
HEADING_SHORT_TEXT_ENABLED = True # 是否启用短中文文本标题识别(最易误判的规则,可单独关闭)
|
||||
HEADING_SHORT_TEXT_MAX_LENGTH = 20 # 短文本最大字符数阈值
|
||||
|
||||
# 表单类型二次校正
|
||||
# MinerU 解析 Word 文档时,某些表单被标记为 text,根据内容特征修正为 table
|
||||
FORM_RECLASSIFY_ENABLED = True # 是否启用 text->table 表单检测
|
||||
FORM_RECLASSIFY_MIN_INDICATORS = 2 # 最少命中几个表单特征指标才校正
|
||||
|
||||
# 分块参数
|
||||
CHUNK_SIZE = 1000
|
||||
|
||||
@@ -29,6 +29,7 @@ RAG 核心引擎
|
||||
|
||||
import os
|
||||
import gc
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
@@ -112,7 +113,7 @@ except ImportError:
|
||||
RERANK_DEVICE = "auto"
|
||||
RERANK_USE_ONNX = False
|
||||
RERANK_BACKEND = "local"
|
||||
RERANK_CLOUD_MODEL = "qwen3-rerank"
|
||||
RERANK_CLOUD_MODEL = "xop3qwen8breranker"
|
||||
RERANK_CLOUD_API_KEY = ""
|
||||
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
||||
RERANK_CLOUD_TIMEOUT = 15
|
||||
@@ -272,8 +273,7 @@ class CloudReranker:
|
||||
body = {
|
||||
"model": self.model,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"top_n": len(documents)
|
||||
"documents": documents
|
||||
}
|
||||
|
||||
session = self._get_session()
|
||||
@@ -1231,6 +1231,7 @@ class RAGEngine:
|
||||
logger.warning(f"扩展连续切片失败: {e}")
|
||||
return {'ids': [], 'documents': [], 'metadatas': []}
|
||||
|
||||
# 扩展同 section 的 text 邻居
|
||||
where_filter = {"$and": [{"source": source}, {"chunk_type": "text"}]}
|
||||
if section:
|
||||
where_filter["$and"].append({"section": section})
|
||||
@@ -1241,6 +1242,20 @@ class RAGEngine:
|
||||
if not neighbors.get('ids') or len(neighbors.get('ids', [])) <= 1:
|
||||
neighbors = _get_neighbors({"$and": [{"source": source}, {"chunk_type": "text"}]})
|
||||
|
||||
# 同时扩展同 section 的 table 邻居(table 切片的 rerank 分数往往偏低,
|
||||
# 但与同 section 的 text 切片属于同一语义单元,不应割裂)
|
||||
table_where = {"$and": [{"source": source}, {"chunk_type": "table"}]}
|
||||
if section:
|
||||
table_where["$and"].append({"section": section})
|
||||
table_neighbors = _get_neighbors(table_where)
|
||||
|
||||
# 当 section 为空时,table 查询只有 source 条件,可能拉入大量无关表格,
|
||||
# 缩小 chunk_index 窗口至 ±1 以降低噪音;有 section 时使用正常窗口
|
||||
if section:
|
||||
_t_before, _t_after = CONTEXT_EXPANSION_BEFORE, CONTEXT_EXPANSION_AFTER
|
||||
else:
|
||||
_t_before, _t_after = 1, 1
|
||||
|
||||
neighbor_rows = []
|
||||
for n_id, n_doc, n_meta in zip(
|
||||
neighbors.get('ids', []),
|
||||
@@ -1253,6 +1268,18 @@ class RAGEngine:
|
||||
if seed_index - CONTEXT_EXPANSION_BEFORE <= n_index <= seed_index + CONTEXT_EXPANSION_AFTER:
|
||||
neighbor_rows.append((n_index, n_id, n_doc, n_meta))
|
||||
|
||||
# 同 section 的 table 邻居也加入扩展范围
|
||||
for n_id, n_doc, n_meta in zip(
|
||||
table_neighbors.get('ids', []),
|
||||
table_neighbors.get('documents', []),
|
||||
table_neighbors.get('metadatas', [])
|
||||
):
|
||||
n_index = self._to_int(n_meta.get('chunk_index'))
|
||||
if n_index is None:
|
||||
continue
|
||||
if seed_index - _t_before <= n_index <= seed_index + _t_after:
|
||||
neighbor_rows.append((n_index, n_id, n_doc, n_meta))
|
||||
|
||||
seed_neighbors_added = 0
|
||||
for n_index, n_id, n_doc, n_meta in sorted(neighbor_rows, key=lambda row: row[0]):
|
||||
if len(items) >= max_chunks:
|
||||
@@ -2041,21 +2068,33 @@ class RAGEngine:
|
||||
"content": (
|
||||
"你是一个严谨的知识库问答助手。"
|
||||
"你必须且只能根据用户提供的【参考资料】回答问题。"
|
||||
"参考资料中每段内容前标有章节路径(━格式),请注意区分不同章节的内容,"
|
||||
"特别当不同章节标题相似或包含相同关键词时,务必根据章节路径准确定位,不要混淆。"
|
||||
"如果参考资料中有答案,必须引用对应内容回答,并在回答末尾标注引用编号(如[1]、[2])。"
|
||||
"如果参考资料中确实没有相关信息,简短说明即可,不要编造或补充资料外的内容。"
|
||||
"禁止使用参考资料以外的知识进行补充或推测。"
|
||||
"【重要-表格处理规则】当用户询问表格、要求展示表格内容时,你必须将参考资料中的 Markdown 表格原样输出(保留 | 分隔符和表格结构),"
|
||||
"不要仅用文字描述表格存在或仅列出章节名称。如果参考资料中多个章节都有表格,"
|
||||
"优先展示与用户问题最相关的表格完整内容。"
|
||||
)
|
||||
})
|
||||
|
||||
# 添加当前问题(带上下文)- 强化指令
|
||||
if context:
|
||||
# 检测用户问题是否涉及表格,加入针对性指令
|
||||
_table_hint = ""
|
||||
# 检测上下文中是否包含 Markdown 表格(数据驱动,无需硬编码关键词)
|
||||
_has_table_in_context = bool(re.search(r'\|.+\|', context)) if context else False
|
||||
if _has_table_in_context:
|
||||
_table_hint = "\n注意:参考资料中包含 Markdown 格式的表格数据,请务必将相关表格以原始 Markdown 表格格式完整展示在回答中,不要仅用文字描述。"
|
||||
|
||||
user_message = f"""【参考资料】
|
||||
{context}
|
||||
|
||||
【用户问题】
|
||||
{query}
|
||||
|
||||
请仔细阅读以上全部参考资料后回答。如果参考资料中包含相关内容,必须引用回答并标注编号。如果资料中没有相关信息,请明确说明。"""
|
||||
请仔细阅读以上全部参考资料后回答。注意参考资料中标有章节路径,请根据章节路径准确定位相关内容。如果参考资料中包含相关内容,必须引用回答并标注编号。如果资料中没有相关信息,请明确说明。{_table_hint}"""
|
||||
else:
|
||||
user_message = query
|
||||
|
||||
|
||||
@@ -83,9 +83,16 @@ class IntentAnalyzer:
|
||||
根据对话历史和当前用户消息,输出一个 JSON 对象,包含以下字段:
|
||||
|
||||
1. **rewritten_query**: 改写后的完整问题
|
||||
- 如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
||||
- **指代消解**:如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
||||
- 例如:"分析一下这两张图片" → "分析一下对话历史中提到的图片"
|
||||
- 如果问题本身已经完整,直接返回原文
|
||||
- **追问补全**:如果问题是省略式追问(省略了上一轮讨论的主题实体),必须补全为完整问题
|
||||
- 判断方法:当前问题缺少主语/宾语,且对话历史中可以推断出省略的实体
|
||||
- 补全方法:从上一轮用户问题中提取主题实体,与追问组合成完整问题
|
||||
- 例如:
|
||||
- 上一轮问"吸烟点C1类是什么区?",追问"有完整表格吗?" → "吸烟点C1类有完整表格吗?"
|
||||
- 上一轮问"三峡工程的投资情况",追问"建设地点在哪?" → "三峡工程的建设地点在哪?"
|
||||
- 上一轮问"货源投放有哪些原则?",追问"具体内容是什么?" → "货源投放原则的具体内容是什么?"
|
||||
- 如果问题本身已经完整且独立,直接返回原文
|
||||
|
||||
2. **use_context**: 布尔值
|
||||
- true: 问题依赖历史对话中的信息,答案已经在历史回答中
|
||||
@@ -105,7 +112,9 @@ class IntentAnalyzer:
|
||||
- 推理类(intent="reasoning"):生成最多2个子查询
|
||||
* 原问题的检索查询
|
||||
* 一个补充角度的检索查询(如原因、背景、影响等),帮助获取更全面的上下文
|
||||
- 其他类(factual/instruction/other):严格只生成1个子查询(原问题)
|
||||
- 其他类(factual/instruction/other):严格只生成1个子查询
|
||||
* 子查询应基于 rewritten_query(改写后的完整问题),而非用户原始输入
|
||||
* 例如:追问"有完整表格吗?"改写为"吸烟点C1类有完整表格吗?"后,子查询应为"吸烟点C1类的完整表格内容"
|
||||
- 不要为同一实体生成语义重叠的查询
|
||||
- 子查询应保持原问题的关键词,长度20-60字符为宜
|
||||
|
||||
@@ -307,7 +316,8 @@ class IntentAnalyzer:
|
||||
|
||||
if cache_emb is not None:
|
||||
cached = cache.get(cache_emb)
|
||||
if cached:
|
||||
# 确保缓存条目是意图分析结果(非 RAG 回答缓存)
|
||||
if cached and cached.get("cache_type") != "rag_answer":
|
||||
logger.info(f"意图分析缓存命中: {cached.get('reason', '')[:50]}")
|
||||
return IntentAnalysis.from_dict(cached)
|
||||
else:
|
||||
@@ -377,9 +387,11 @@ class IntentAnalyzer:
|
||||
intent=intent_type
|
||||
)
|
||||
|
||||
# 存入语义缓存
|
||||
# 存入语义缓存(标记类型,避免与 RAG 回答缓存混淆)
|
||||
if cache and cache_emb is not None:
|
||||
cache.set(cache_emb, analysis.to_dict())
|
||||
cache_data = analysis.to_dict()
|
||||
cache_data["cache_type"] = "intent_analysis"
|
||||
cache.set(cache_emb, cache_data)
|
||||
|
||||
# 存入精确匹配缓存
|
||||
if len(self._exact_cache) < self._exact_cache_max:
|
||||
@@ -437,6 +449,7 @@ class IntentAnalyzer:
|
||||
if not history:
|
||||
return "(无历史对话)"
|
||||
|
||||
import re
|
||||
parts = []
|
||||
|
||||
# 提取最近 3 轮对话
|
||||
@@ -445,6 +458,7 @@ class IntentAnalyzer:
|
||||
for msg in recent_history:
|
||||
role = "用户" if msg.get("role") == "user" else "助手"
|
||||
content = msg.get("content", "")
|
||||
original_content = content # 保留原始内容用于结构化提取
|
||||
|
||||
# 截断过长的内容
|
||||
if len(content) > 500:
|
||||
@@ -452,9 +466,10 @@ class IntentAnalyzer:
|
||||
|
||||
parts.append(f"【{role}】{content}")
|
||||
|
||||
# 提取图片信息
|
||||
# 提取结构化信息(从 assistant 消息中提取章节、表格、来源等)
|
||||
metadata = msg.get("metadata", {})
|
||||
if isinstance(metadata, dict):
|
||||
# 已有的图片提取
|
||||
images = metadata.get("images", [])
|
||||
if images:
|
||||
for img in images[:3]:
|
||||
@@ -463,6 +478,43 @@ class IntentAnalyzer:
|
||||
img_type = img.get("type", "图片")
|
||||
parts.append(f" └─ {img_type}: {desc}")
|
||||
|
||||
# 来源文件提取
|
||||
sources = metadata.get("sources", [])
|
||||
if sources:
|
||||
source_names = []
|
||||
for s in sources[:3]:
|
||||
if isinstance(s, dict):
|
||||
name = s.get("source", "") or s.get("name", "")
|
||||
if name:
|
||||
source_names.append(name)
|
||||
elif isinstance(s, str):
|
||||
source_names.append(s)
|
||||
if source_names:
|
||||
parts.append(f" └─ 来源文件: {', '.join(source_names)}")
|
||||
|
||||
# collections(检索知识库)提取
|
||||
colls = metadata.get("collections", [])
|
||||
if colls:
|
||||
parts.append(f" └─ 检索知识库: {', '.join(colls)}")
|
||||
|
||||
# 从 assistant 原始内容中提取章节路径和表格结构
|
||||
if role == "助手" and original_content:
|
||||
# 提取章节路径:━ xxx ━ 格式
|
||||
sections = re.findall(r'━\s*(.+?)\s*━', original_content)
|
||||
if sections:
|
||||
unique_sections = list(dict.fromkeys(sections)) # 去重保序
|
||||
parts.append(f" └─ 涉及章节: {'; '.join(unique_sections[:3])}")
|
||||
|
||||
# 提取表格列名:| A | B | C | 格式的表头行
|
||||
table_headers = re.findall(r'^\|\s*(.+?)\s*\|', original_content, re.MULTILINE)
|
||||
if table_headers:
|
||||
# 取第一个表格的列名
|
||||
first_header = table_headers[0]
|
||||
cols = [c.strip() for c in first_header.split('|') if c.strip()]
|
||||
# 排除分隔符行(--- 格式)
|
||||
if cols and not all(re.match(r'^[-:]+$', c) for c in cols):
|
||||
parts.append(f" └─ 含表格,列名: {', '.join(cols[:6])}")
|
||||
|
||||
# 添加图片上下文
|
||||
if context_images:
|
||||
parts.append("\n【上下文中的图片】")
|
||||
|
||||
@@ -352,7 +352,7 @@ def quick_yes_no(
|
||||
if keywords is None:
|
||||
keywords = ["是", "需要", "yes", "true"]
|
||||
|
||||
result = call_llm(client, prompt, model, temperature=0, max_tokens=10)
|
||||
result = call_llm(client, prompt, model, temperature=0, max_tokens=128)
|
||||
if result is None:
|
||||
return False
|
||||
|
||||
|
||||
@@ -393,6 +393,11 @@ class KnowledgeBaseManager(
|
||||
if len(chunks) < 2:
|
||||
return chunks
|
||||
|
||||
# 检测页码是否可靠:若所有 chunk 的 page_start 相同(如 Word 文档 page_idx 全为 0),
|
||||
# 则页码信息不可用,需要启用降级合并规则
|
||||
page_values = set(getattr(c, 'page_start', 0) for c in chunks)
|
||||
pages_unavailable = len(page_values) <= 1
|
||||
|
||||
merged_chunks = []
|
||||
i = 0
|
||||
merge_count = 0
|
||||
@@ -405,6 +410,7 @@ class KnowledgeBaseManager(
|
||||
# 查找下一个表格(跳过中间的"续表"文本)
|
||||
next_table_idx = None
|
||||
next_chunk = None
|
||||
intermediate_texts = [] # 收集中间文本用于降级判断
|
||||
|
||||
for j in range(i + 1, min(i + 4, len(chunks))): # 最多向前看3个切片
|
||||
candidate = chunks[j]
|
||||
@@ -419,7 +425,11 @@ class KnowledgeBaseManager(
|
||||
elif candidate_type == 'text' and ('续表' in candidate_title or '续表' in candidate_content):
|
||||
# 遇到"续表"文本,继续查找下一个表格
|
||||
continue
|
||||
elif candidate_type not in ('text',):
|
||||
elif candidate_type == 'text':
|
||||
# 非"续表"文本,收集后停止查找
|
||||
intermediate_texts.append(candidate)
|
||||
break
|
||||
else:
|
||||
# 遇到非文本类型,停止查找
|
||||
break
|
||||
|
||||
@@ -439,12 +449,22 @@ class KnowledgeBaseManager(
|
||||
# 获取内容(用于检测"续表")
|
||||
next_content = getattr(next_chunk, 'content', '')
|
||||
|
||||
# 通用/无意义标题集合,这些标题不能用于"标题相似"判定
|
||||
_GENERIC_TITLES = {'表格', 'table', '表格', ''}
|
||||
|
||||
# 判断是否为跨页表格
|
||||
is_cross_page = False
|
||||
|
||||
# 规则1: 页码连续(如果页码有效)
|
||||
page_valid = curr_page_end > 0 and next_page_start > 0
|
||||
if page_valid and curr_page_end + 1 == next_page_start:
|
||||
# 页码连续时,还需标题匹配或为通用标题才合并
|
||||
# 避免把不同页面上不相关的表格错误合并
|
||||
if curr_title == next_title or curr_title in _GENERIC_TITLES and next_title in _GENERIC_TITLES:
|
||||
is_cross_page = True
|
||||
elif curr_title and next_title:
|
||||
clean_next_r1 = next_title.replace('续表', '').strip()
|
||||
if curr_title in clean_next_r1 or clean_next_r1 in curr_title:
|
||||
is_cross_page = True
|
||||
|
||||
# 规则2: 第二个表格标题或内容包含"续表"
|
||||
@@ -452,11 +472,36 @@ class KnowledgeBaseManager(
|
||||
is_cross_page = True
|
||||
|
||||
# 规则3: 标题相似(去掉"续表"后比较)
|
||||
elif curr_title and next_title:
|
||||
# 排除通用标题(如"表格"),防止把所有标题为"表格"的相邻表格都误合并
|
||||
elif (curr_title and next_title
|
||||
and curr_title not in _GENERIC_TITLES
|
||||
and next_title not in _GENERIC_TITLES):
|
||||
clean_next = next_title.replace('续表', '').strip()
|
||||
if curr_title in clean_next or clean_next in curr_title:
|
||||
if clean_next and (curr_title in clean_next or clean_next in curr_title):
|
||||
is_cross_page = True
|
||||
|
||||
# 规则4(降级): 页码不可用(如 Word 文档 page_idx 全为 0)
|
||||
# 仅当页码信息缺失时才启用此规则,避免 PDF 正常页码时被误合并
|
||||
if (not is_cross_page
|
||||
and pages_unavailable
|
||||
and curr_title in _GENERIC_TITLES
|
||||
and next_title in _GENERIC_TITLES):
|
||||
# 检查中间文本是否暗示跨页延续(空、短文本、续表标记等)
|
||||
has_separating_content = False
|
||||
for text_chunk in intermediate_texts:
|
||||
tc = (getattr(text_chunk, 'content', '') or '').strip()
|
||||
tt = (getattr(text_chunk, 'title', '') or '').strip()
|
||||
if not tc:
|
||||
continue # 空文本不算分隔
|
||||
if '续表' in tc or '续表' in tt:
|
||||
continue # 续表标记,说明是跨页
|
||||
# 有实质性中间内容(如分类标题"A3类:xxx"),不合并
|
||||
has_separating_content = True
|
||||
break
|
||||
if not has_separating_content:
|
||||
is_cross_page = True
|
||||
logger.debug(f"降级合并(页码不可用): '{curr_title}' + '{next_title}'")
|
||||
|
||||
if is_cross_page:
|
||||
# 执行合并
|
||||
merge_count += 1
|
||||
@@ -469,14 +514,27 @@ class KnowledgeBaseManager(
|
||||
# 合并两个表格的 HTML
|
||||
current.table_html = curr_html + '\n' + next_html
|
||||
|
||||
# 合并 image_path 到 images
|
||||
# 合并 image_path 和嵌入图片到 images
|
||||
curr_img = getattr(current, 'image_path', None)
|
||||
next_img = getattr(next_chunk, 'image_path', None)
|
||||
merged_images = []
|
||||
if curr_img:
|
||||
curr_images = getattr(current, 'images', None) or []
|
||||
next_images = getattr(next_chunk, 'images', None) or []
|
||||
|
||||
# 合并两个表格的所有图片(image_path + 嵌入图片)
|
||||
merged_images = list(curr_images) # 保留当前表格的嵌入图片
|
||||
# 添加 image_path 图片(如果不在列表中)
|
||||
existing_ids = {img.get('id', '') for img in merged_images if isinstance(img, dict)}
|
||||
if curr_img and curr_img not in existing_ids:
|
||||
merged_images.append({'id': curr_img, 'page': curr_page_end})
|
||||
if next_img:
|
||||
existing_ids.add(curr_img)
|
||||
for img in next_images: # 添加下一个表格的嵌入图片
|
||||
img_id = img.get('id', '') if isinstance(img, dict) else ''
|
||||
if img_id and img_id not in existing_ids:
|
||||
merged_images.append(img)
|
||||
existing_ids.add(img_id)
|
||||
if next_img and next_img not in existing_ids:
|
||||
merged_images.append({'id': next_img, 'page': next_page_start})
|
||||
|
||||
if merged_images:
|
||||
current.images = merged_images
|
||||
# 保留第一个图片作为主 image_path
|
||||
@@ -525,7 +583,7 @@ class KnowledgeBaseManager(
|
||||
try:
|
||||
from config import get_llm_client, DASHSCOPE_MODEL
|
||||
client = get_llm_client()
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=100)
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
||||
return summary.strip() if summary else ""
|
||||
except Exception as e:
|
||||
logger.warning(f"生成表格摘要失败: {e}")
|
||||
@@ -584,7 +642,7 @@ class KnowledgeBaseManager(
|
||||
]
|
||||
}
|
||||
],
|
||||
max_tokens=200
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
description = response.choices[0].message.content
|
||||
|
||||
@@ -283,7 +283,7 @@ class KnowledgeBaseRouter:
|
||||
content = call_llm(
|
||||
self.llm_client, prompt, MODEL,
|
||||
temperature=0.1,
|
||||
max_tokens=100
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
if content is None:
|
||||
|
||||
347
parsers/heading_rules.py
Normal file
347
parsers/heading_rules.py
Normal file
@@ -0,0 +1,347 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
标题识别规则引擎
|
||||
|
||||
将 _detect_heading_level 的硬编码正则提取为可配置的规则列表。
|
||||
规则按优先级从高到低排序,第一个匹配即返回。
|
||||
|
||||
MinerU 解析 DOCX 等 Office 格式时通常不提供 text_level(全部为 0),
|
||||
此时需要启发式识别标题层级。本模块提供可配置的规则引擎替代原来的
|
||||
硬编码 if-elif 链。
|
||||
|
||||
设计要点:
|
||||
- HeadingRule 数据类支持正向匹配(pattern)和反向排除(exclude_pattern)
|
||||
- 长度约束(min_length / max_length)可精确控制匹配范围
|
||||
- 规则可单独禁用(enabled=False),便于调试
|
||||
- 全局单例通过 config.py 覆盖默认值
|
||||
|
||||
MinerU v2 格式备注:
|
||||
content_list_v2.json 中的 paragraph_content 包含 style=["bold"] 信息,
|
||||
layout.json 中的 spans 也有 style 信息。这些信息比正则匹配 **加粗** 更可靠,
|
||||
但当前代码使用 v1 格式(content_list.json),暂不利用 v2 的 style。
|
||||
HeadingRuleEngine.detect() 签名预留了 style 参数,未来切换到 v2 格式后
|
||||
可直接利用 style 信息辅助判断。
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, List, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeadingRule:
|
||||
"""
|
||||
标题识别规则
|
||||
|
||||
每条规则定义一个文本模式到标题级别的映射。
|
||||
规则引擎按列表顺序逐条匹配,第一个命中即返回。
|
||||
|
||||
Attributes:
|
||||
pattern: 编译后的正则(match 语义,从文本开头匹配)
|
||||
level: 匹配时返回的标题级别 (1=h1, 2=h2, 3=h3)
|
||||
name: 规则名称(用于日志和配置覆盖)
|
||||
max_length: 文本最大长度,0=不限
|
||||
min_length: 文本最小长度,0=不限
|
||||
enabled: 是否启用
|
||||
exclude_pattern: 匹配此模式则排除(反向过滤)
|
||||
|
||||
Example:
|
||||
>>> rule = HeadingRule(
|
||||
... pattern=re.compile(r'^第[一二三四五六七八九十百千万]+[章节篇部]'),
|
||||
... level=1,
|
||||
... name="chinese_chapter",
|
||||
... )
|
||||
>>> rule.match("第一章 总则")
|
||||
1
|
||||
>>> rule.match("这是正文")
|
||||
0
|
||||
"""
|
||||
|
||||
pattern: re.Pattern
|
||||
level: int
|
||||
name: str
|
||||
max_length: int = 0
|
||||
min_length: int = 0
|
||||
enabled: bool = True
|
||||
exclude_pattern: Optional[re.Pattern] = None
|
||||
|
||||
def match(self, text: str) -> int:
|
||||
"""
|
||||
检查文本是否匹配此规则
|
||||
|
||||
Args:
|
||||
text: 待检测文本(调用前应已 strip)
|
||||
|
||||
Returns:
|
||||
标题级别,0 表示不匹配
|
||||
"""
|
||||
if not self.enabled:
|
||||
return 0
|
||||
if self.min_length > 0 and len(text) < self.min_length:
|
||||
return 0
|
||||
if self.max_length > 0 and len(text) > self.max_length:
|
||||
return 0
|
||||
if self.exclude_pattern and self.exclude_pattern.search(text):
|
||||
return 0
|
||||
if self.pattern.match(text):
|
||||
return self.level
|
||||
return 0
|
||||
|
||||
|
||||
# 默认规则列表(按优先级从高到低)
|
||||
#
|
||||
# 注意事项:
|
||||
# - 数字三级标题 (1.1.1) 必须在二级 (1.1) 之前,因为 1.1.1 也匹配 ^\d+\.\d+
|
||||
# - short_chinese_heading 是最宽泛的规则,放在最后作为兜底
|
||||
# - 第 9 条规则相比原版增加了 exclude_pattern,排除以句末标点结尾的短文本
|
||||
DEFAULT_HEADING_RULES: List[HeadingRule] = [
|
||||
# 1. 中文章节标题 -> h1
|
||||
# 匹配:第一章、第二章、第十节、第三篇 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^第[一二三四五六七八九十百千万]+[章节篇部]'),
|
||||
level=1,
|
||||
name="chinese_chapter",
|
||||
),
|
||||
# 2. 中文条款编号 -> h2
|
||||
# 匹配:第一条、第三款 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^第[一二三四五六七八九十百千万]+[条款]'),
|
||||
level=2,
|
||||
name="chinese_article",
|
||||
),
|
||||
# 3. 数字三级标题 -> h3(必须在二级之前匹配)
|
||||
# 匹配:1.1.1 背景、2.3.4 方案 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^\d+\.\d+\.\d+[\.、\s]'),
|
||||
level=3,
|
||||
name="numeric_level3",
|
||||
max_length=100,
|
||||
),
|
||||
# 4. 数字二级标题 -> h2(必须在一级之前匹配)
|
||||
# 匹配:1.1 背景、2.3 方案 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^\d+\.\d+[\.、\s]'),
|
||||
level=2,
|
||||
name="numeric_level2",
|
||||
max_length=80,
|
||||
),
|
||||
# 5. 数字一级标题 -> h1
|
||||
# 匹配:1. 概述、2、背景 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^\d+[\.、\s]'),
|
||||
level=1,
|
||||
name="numeric_level1",
|
||||
max_length=50,
|
||||
),
|
||||
# 6. 英文章节标题 -> h1
|
||||
# 匹配:Chapter 1、Section 2、Part 3 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^(Chapter|Section|Part|Chapter\s+\d+|Section\s+\d+)', re.IGNORECASE),
|
||||
level=1,
|
||||
name="english_chapter",
|
||||
),
|
||||
# 7. 分类标题 -> h3(必须在 bold_short_text 之前,否则 **A2类:** 会被加粗规则抢先匹配)
|
||||
# 匹配:A1类:公园、**A2类**:各类卫生医疗机构、**B1类:** 道路 等
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^\*{0,2}[A-Z]\d+[类類]\*{0,2}[::]'),
|
||||
level=3,
|
||||
name="category_heading",
|
||||
),
|
||||
# 8. 加粗短文本 -> h2
|
||||
# 匹配:**重要通知**、**概述** 等(Markdown 加粗标记)
|
||||
# 注意:**A2类:** 已被分类标题规则优先匹配,不会误判为 h2
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'^\*\*.+\*\*$'),
|
||||
level=2,
|
||||
name="bold_short_text",
|
||||
max_length=50,
|
||||
),
|
||||
# 9. 短中文文本 -> h2(替代原"任何 <20 字符含中文"规则)
|
||||
# 关键改进:排除以句末标点结尾的文本
|
||||
# 原规则将 "这是一段正文。" 也识别为 h2,导致大量误判
|
||||
# 新规则:包含中文 + 长度 2-20 + 不以句末标点结尾 → h2
|
||||
HeadingRule(
|
||||
pattern=re.compile(r'[一-鿿]'),
|
||||
level=2,
|
||||
name="short_chinese_heading",
|
||||
max_length=20,
|
||||
min_length=2,
|
||||
exclude_pattern=re.compile(r'[。!?;…]$'),
|
||||
enabled=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class HeadingRuleEngine:
|
||||
"""
|
||||
标题识别规则引擎
|
||||
|
||||
按规则列表顺序逐条匹配,第一个命中即返回标题级别。
|
||||
支持从 config.py 加载自定义规则或覆盖默认规则参数。
|
||||
|
||||
Example:
|
||||
>>> engine = HeadingRuleEngine()
|
||||
>>> engine.detect("第一章 总则")
|
||||
(1, 'chinese_chapter')
|
||||
>>> engine.detect("这是普通正文。")
|
||||
(0, None)
|
||||
"""
|
||||
|
||||
def __init__(self, rules: Optional[List[HeadingRule]] = None) -> None:
|
||||
"""
|
||||
Args:
|
||||
rules: 规则列表,None 则使用默认规则的深拷贝
|
||||
"""
|
||||
if rules is not None:
|
||||
self.rules: List[HeadingRule] = rules
|
||||
else:
|
||||
import copy
|
||||
self.rules = copy.deepcopy(DEFAULT_HEADING_RULES)
|
||||
|
||||
def detect(self, text: str, style: Optional[List[str]] = None) -> Tuple[int, Optional[str]]:
|
||||
"""
|
||||
检测文本的标题级别
|
||||
|
||||
Args:
|
||||
text: 待检测文本
|
||||
style: MinerU v2 格式中的 style 信息(如 ["bold"]),
|
||||
当文本标记为 bold 且较短时,可直接判定为标题,
|
||||
无需依赖 Markdown **...** 标记。
|
||||
|
||||
Returns:
|
||||
(level, rule_name): 标题级别和匹配的规则名
|
||||
level=0 表示不是标题
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return 0, None
|
||||
|
||||
# v2 style 信息:如果文本标记为 bold 且较短,优先尝试加粗规则
|
||||
if style and 'bold' in style and 2 <= len(text) <= 50:
|
||||
# 先检查是否匹配更高优先级的分类标题规则
|
||||
for rule in self.rules:
|
||||
if rule.name == 'category_heading' and rule.enabled:
|
||||
level = rule.match(text)
|
||||
if level > 0:
|
||||
logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h{level} (规则: {rule.name})")
|
||||
return level, rule.name
|
||||
|
||||
# 再检查是否匹配中文章节/条款等高优先级规则
|
||||
for rule in self.rules:
|
||||
if rule.name in ('chinese_chapter', 'chinese_article', 'numeric_level3',
|
||||
'numeric_level2', 'numeric_level1', 'english_chapter') and rule.enabled:
|
||||
level = rule.match(text)
|
||||
if level > 0:
|
||||
logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h{level} (规则: {rule.name})")
|
||||
return level, rule.name
|
||||
|
||||
# 否则作为加粗短文本 → h2(与 bold_short_text 规则对齐,但不依赖 **...** 标记)
|
||||
logger.debug(f"标题识别(v2 style): '{text[:30]}' -> h2 (规则: bold_short_text_via_style)")
|
||||
return 2, 'bold_short_text'
|
||||
|
||||
# 常规规则匹配(v1 格式或无 style 信息时)
|
||||
for rule in self.rules:
|
||||
level = rule.match(text)
|
||||
if level > 0:
|
||||
logger.debug(f"标题识别: '{text[:30]}' -> h{level} (规则: {rule.name})")
|
||||
return level, rule.name
|
||||
|
||||
return 0, None
|
||||
|
||||
|
||||
# ==================== 全局单例 ====================
|
||||
|
||||
_engine: Optional[HeadingRuleEngine] = None
|
||||
|
||||
|
||||
def get_heading_engine() -> HeadingRuleEngine:
|
||||
"""获取全局标题识别引擎(延迟初始化,线程安全)"""
|
||||
global _engine
|
||||
if _engine is None:
|
||||
_engine = _create_engine_from_config()
|
||||
return _engine
|
||||
|
||||
|
||||
def _create_engine_from_config() -> HeadingRuleEngine:
|
||||
"""
|
||||
从 config 创建引擎(支持配置覆盖)
|
||||
|
||||
优先级:
|
||||
1. config.HEADING_RULES_CONFIG 不为 None → 使用自定义规则
|
||||
2. config 细粒度参数覆盖默认规则(如 HEADING_SHORT_TEXT_ENABLED)
|
||||
3. 使用默认规则
|
||||
"""
|
||||
# 尝试加载完整自定义规则
|
||||
try:
|
||||
from config import HEADING_RULES_CONFIG
|
||||
if HEADING_RULES_CONFIG is not None:
|
||||
rules = _build_rules_from_config(HEADING_RULES_CONFIG)
|
||||
logger.info(f"使用自定义标题规则: {len(rules)} 条")
|
||||
return HeadingRuleEngine(rules)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 使用默认规则,应用细粒度配置覆盖
|
||||
rules = list(DEFAULT_HEADING_RULES)
|
||||
try:
|
||||
from config import HEADING_SHORT_TEXT_ENABLED
|
||||
for rule in rules:
|
||||
if rule.name == "short_chinese_heading":
|
||||
rule.enabled = HEADING_SHORT_TEXT_ENABLED
|
||||
logger.debug(f"配置覆盖: short_chinese_heading.enabled={HEADING_SHORT_TEXT_ENABLED}")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from config import HEADING_SHORT_TEXT_MAX_LENGTH
|
||||
for rule in rules:
|
||||
if rule.name == "short_chinese_heading":
|
||||
rule.max_length = HEADING_SHORT_TEXT_MAX_LENGTH
|
||||
logger.debug(f"配置覆盖: short_chinese_heading.max_length={HEADING_SHORT_TEXT_MAX_LENGTH}")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return HeadingRuleEngine(rules)
|
||||
|
||||
|
||||
def _build_rules_from_config(config: list) -> List[HeadingRule]:
|
||||
"""
|
||||
从配置字典列表构建规则列表
|
||||
|
||||
Args:
|
||||
config: 规则配置列表,每项为 dict,包含:
|
||||
- pattern (str): 正则表达式字符串
|
||||
- level (int): 标题级别
|
||||
- name (str): 规则名称
|
||||
- max_length (int, 可选): 文本最大长度
|
||||
- min_length (int, 可选): 文本最小长度
|
||||
- enabled (bool, 可选): 是否启用
|
||||
- exclude_pattern (str, 可选): 排除正则
|
||||
|
||||
Returns:
|
||||
规则列表
|
||||
"""
|
||||
rules = []
|
||||
for item in config:
|
||||
exclude = None
|
||||
if 'exclude_pattern' in item:
|
||||
exclude = re.compile(item['exclude_pattern'])
|
||||
rules.append(HeadingRule(
|
||||
pattern=re.compile(item['pattern']),
|
||||
level=item['level'],
|
||||
name=item['name'],
|
||||
max_length=item.get('max_length', 0),
|
||||
min_length=item.get('min_length', 0),
|
||||
enabled=item.get('enabled', True),
|
||||
exclude_pattern=exclude,
|
||||
))
|
||||
return rules
|
||||
|
||||
|
||||
def reset_heading_engine() -> None:
|
||||
"""重置引擎(用于测试)"""
|
||||
global _engine
|
||||
_engine = None
|
||||
@@ -286,14 +286,138 @@ def parse_with_mineru_online(
|
||||
raise RuntimeError(f"MinerU 在线 API 调用失败: {e}")
|
||||
|
||||
|
||||
def _parse_v2_content_list(v2_data: list) -> list:
|
||||
"""
|
||||
将 MinerU v2 嵌套格式转换为 v1 兼容的扁平 content_list
|
||||
|
||||
v2 格式: [[page0_items], [page1_items], ...]
|
||||
v1 格式: [item, item, ...] 扁平列表
|
||||
|
||||
转换规则:
|
||||
- paragraph → text(拼接 paragraph_content,提取 style 信息)
|
||||
- table → table(保留 table_body/html)
|
||||
- title → text(提取 level 信息)
|
||||
- page_header / page_footer / page_number → 过滤掉
|
||||
|
||||
Args:
|
||||
v2_data: v2 格式的嵌套列表
|
||||
|
||||
Returns:
|
||||
v1 兼容的扁平列表,额外包含 _v2_styles / _v2_table_type 字段
|
||||
"""
|
||||
flat_list: List[Dict] = []
|
||||
|
||||
for page_idx, page_items in enumerate(v2_data):
|
||||
if not isinstance(page_items, list):
|
||||
continue
|
||||
for item in page_items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
v2_type: str = item.get('type', '')
|
||||
content = item.get('content', {})
|
||||
|
||||
# 过滤噪音类型(页眉、页脚、页码)
|
||||
if v2_type in ('page_header', 'page_footer', 'page_number'):
|
||||
continue
|
||||
|
||||
if v2_type == 'paragraph':
|
||||
# 拼接段落文本,收集样式信息
|
||||
text_parts: List[str] = []
|
||||
has_bold = False
|
||||
para_content = content.get('paragraph_content', []) if isinstance(content, dict) else []
|
||||
for part in para_content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
part_text = part.get('content', '')
|
||||
text_parts.append(part_text)
|
||||
if 'bold' in part.get('style', []):
|
||||
has_bold = True
|
||||
|
||||
full_text = ''.join(text_parts).strip()
|
||||
if not full_text:
|
||||
continue
|
||||
|
||||
flat_item: Dict[str, Any] = {
|
||||
'type': 'text',
|
||||
'text': full_text,
|
||||
'content': full_text,
|
||||
'page_idx': page_idx,
|
||||
'bbox': item.get('bbox', []),
|
||||
'text_level': 0,
|
||||
'_v2_styles': ['bold'] if has_bold else [],
|
||||
}
|
||||
flat_list.append(flat_item)
|
||||
|
||||
elif v2_type == 'table':
|
||||
flat_item = {
|
||||
'type': 'table',
|
||||
'page_idx': page_idx,
|
||||
'bbox': item.get('bbox', []),
|
||||
'html': content.get('html', '') if isinstance(content, dict) else '',
|
||||
'table_body': content.get('html', '') if isinstance(content, dict) else '',
|
||||
'caption': content.get('caption', '') if isinstance(content, dict) else '',
|
||||
'_v2_table_type': content.get('table_type', '') if isinstance(content, dict) else '',
|
||||
}
|
||||
flat_list.append(flat_item)
|
||||
|
||||
elif v2_type == 'title':
|
||||
# v2 title 项有 level 信息
|
||||
level = content.get('level', 0) if isinstance(content, dict) else 0
|
||||
text_parts = []
|
||||
para_content = content.get('paragraph_content', []) if isinstance(content, dict) else []
|
||||
for part in para_content:
|
||||
if isinstance(part, dict):
|
||||
text_parts.append(part.get('content', ''))
|
||||
full_text = ''.join(text_parts).strip()
|
||||
|
||||
if full_text:
|
||||
flat_item = {
|
||||
'type': 'text',
|
||||
'text': full_text,
|
||||
'content': full_text,
|
||||
'page_idx': page_idx,
|
||||
'bbox': item.get('bbox', []),
|
||||
'text_level': level,
|
||||
'_v2_styles': ['bold'],
|
||||
}
|
||||
flat_list.append(flat_item)
|
||||
|
||||
elif v2_type in ('image', 'chart'):
|
||||
flat_item = {
|
||||
'type': v2_type,
|
||||
'page_idx': page_idx,
|
||||
'bbox': item.get('bbox', []),
|
||||
'img_path': content.get('img_path', '') if isinstance(content, dict) else '',
|
||||
'image_path': content.get('img_path', '') if isinstance(content, dict) else '',
|
||||
'caption': content.get('caption', '') if isinstance(content, dict) else '',
|
||||
}
|
||||
flat_list.append(flat_item)
|
||||
|
||||
elif v2_type == 'equation':
|
||||
flat_item = {
|
||||
'type': 'equation',
|
||||
'page_idx': page_idx,
|
||||
'bbox': item.get('bbox', []),
|
||||
'content': content.get('latex', '') if isinstance(content, dict) else '',
|
||||
'text': content.get('latex', '') if isinstance(content, dict) else '',
|
||||
'latex': content.get('latex', '') if isinstance(content, dict) else '',
|
||||
'img_path': content.get('img_path', '') if isinstance(content, dict) else '',
|
||||
}
|
||||
flat_list.append(flat_item)
|
||||
|
||||
logger.info(f"v2 格式转换: {len(v2_data)} 页 → {len(flat_list)} 项(已过滤噪音类型)")
|
||||
return flat_list
|
||||
|
||||
|
||||
def _parse_mineru_online_zip(zip_content: bytes, file_path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
解析 MinerU 在线 API 返回的 zip 包
|
||||
|
||||
zip 包结构:
|
||||
zip 包结构:
|
||||
- full.md - Markdown 解析结果
|
||||
- *_content_list.json - 内容列表(扁平格式,优先使用)
|
||||
- *_content_list_v2.json - 内容列表(嵌套格式)
|
||||
- *_content_list.json - 内容列表(v1 扁平格式)
|
||||
- *_content_list_v2.json - 内容列表(v2 嵌套格式,含 style 信息)
|
||||
- images/ - 图片目录
|
||||
|
||||
Args:
|
||||
@@ -306,23 +430,44 @@ def _parse_mineru_online_zip(zip_content: bytes, file_path: Path) -> Dict[str, A
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
# 读取格式偏好配置
|
||||
try:
|
||||
from config import MINERU_PREFER_V2
|
||||
except ImportError:
|
||||
MINERU_PREFER_V2 = True # 默认优先 v2
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(zip_content), 'r') as zf:
|
||||
# 列出所有文件
|
||||
file_list = zf.namelist()
|
||||
logger.debug(f"zip 包内容: {file_list}")
|
||||
|
||||
# 查找 content_list.json(优先使用扁平格式)
|
||||
# 查找 content_list 文件(v2 含 style 信息,优先使用)
|
||||
content_list_path = None
|
||||
is_v2 = False
|
||||
|
||||
if MINERU_PREFER_V2:
|
||||
# 优先使用 v2 格式(含 style 信息)
|
||||
for f in file_list:
|
||||
if f.endswith('_content_list_v2.json'):
|
||||
content_list_path = f
|
||||
is_v2 = True
|
||||
break
|
||||
if not content_list_path:
|
||||
for f in file_list:
|
||||
if f.endswith('_content_list.json') and not f.endswith('_v2.json'):
|
||||
content_list_path = f
|
||||
break
|
||||
else:
|
||||
# 优先使用 v1 格式
|
||||
for f in file_list:
|
||||
# 优先使用 content_list.json(扁平格式)
|
||||
if f.endswith('_content_list.json') and not f.endswith('_v2.json'):
|
||||
content_list_path = f
|
||||
break
|
||||
# 如果没有找到,再尝试 v2 格式
|
||||
if not content_list_path:
|
||||
for f in file_list:
|
||||
if f.endswith('_content_list_v2.json'):
|
||||
content_list_path = f
|
||||
is_v2 = True
|
||||
break
|
||||
|
||||
# 查找 markdown 文件
|
||||
@@ -337,7 +482,13 @@ def _parse_mineru_online_zip(zip_content: bytes, file_path: Path) -> Dict[str, A
|
||||
if content_list_path:
|
||||
with zf.open(content_list_path) as f:
|
||||
content_list = json.load(f)
|
||||
logger.info(f"读取 content_list: {len(content_list)} 项, 来源: {content_list_path}")
|
||||
|
||||
# v2 格式需要转换为 v1 兼容的扁平列表
|
||||
if is_v2 and isinstance(content_list, list) and content_list and isinstance(content_list[0], list):
|
||||
content_list = _parse_v2_content_list(content_list)
|
||||
logger.info(f"v2 格式已转换为扁平列表,共 {len(content_list)} 项")
|
||||
|
||||
logger.info(f"读取 content_list: {len(content_list)} 项, 格式={'v2' if is_v2 else 'v1'}, 来源: {content_list_path}")
|
||||
|
||||
# 读取 markdown
|
||||
markdown_content = ""
|
||||
@@ -424,10 +575,14 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any]
|
||||
|
||||
if item_type == "text":
|
||||
text = item.get("content", "") or item.get("text", "")
|
||||
v2_styles = item.get("_v2_styles", [])
|
||||
|
||||
# 启发式标题识别
|
||||
if text_level == 0:
|
||||
text_level = _detect_heading_level(text)
|
||||
from parsers.heading_rules import get_heading_engine
|
||||
engine = get_heading_engine()
|
||||
detected_level, _ = engine.detect(text, style=v2_styles)
|
||||
text_level = detected_level
|
||||
|
||||
title = ""
|
||||
if text_level > 0:
|
||||
@@ -471,7 +626,8 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any]
|
||||
else:
|
||||
md_table = ""
|
||||
|
||||
table_images = extract_images_from_markdown(md_table) if md_table else []
|
||||
# 从原始 HTML 提取嵌入图片(md_table 经 get_text 转换后已丢失 <img> 标签)
|
||||
table_images = extract_images_from_markdown(table_body) if table_body else []
|
||||
|
||||
chunk = MinerUChunk(
|
||||
content=table_caption or "表格",
|
||||
@@ -558,8 +714,14 @@ def _parse_mineru_online_result(result: Dict, file_path: Path) -> Dict[str, Any]
|
||||
min_merge = 100
|
||||
max_size = 1200
|
||||
|
||||
# 表单类型二次校正(在 _post_process_chunks 之前,因为 table 不参与合并)
|
||||
_reclassify_text_chunks(chunks)
|
||||
|
||||
chunks = _post_process_chunks(chunks, min_merge_size=min_merge, max_chunk_size=max_size)
|
||||
|
||||
# 验证分类标题是否被规则引擎正确识别(安全网,仅告警不修改)
|
||||
_validate_category_section_paths(chunks)
|
||||
|
||||
return {
|
||||
'markdown': "\n".join(markdown_parts) if markdown_parts else markdown_content,
|
||||
'chunks': chunks,
|
||||
@@ -697,16 +859,18 @@ def parse_with_mineru(
|
||||
logger.error(f"MinerU 解析失败: {e}")
|
||||
raise
|
||||
finally:
|
||||
# 清理临时目录
|
||||
清理临时目录
|
||||
if cleanup_output and os.path.exists(output_dir):
|
||||
shutil.rmtree(output_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _detect_heading_level(text: str) -> int:
|
||||
"""
|
||||
启发式标题识别
|
||||
启发式标题识别(规则引擎版)
|
||||
|
||||
用于 MinerU 解析 DOCX 等 Office 格式时不提供 text_level 的情况。
|
||||
当 MinerU 解析 DOCX 等 Office 格式时不提供 text_level 时使用。
|
||||
规则按优先级从高到低匹配,第一个命中即返回。
|
||||
|
||||
规则定义见 parsers/heading_rules.py,可通过 config.py 覆盖。
|
||||
|
||||
Args:
|
||||
text: 文本内容
|
||||
@@ -714,57 +878,91 @@ def _detect_heading_level(text: str) -> int:
|
||||
Returns:
|
||||
标题级别 (0=正文, 1=h1, 2=h2, 3=h3)
|
||||
"""
|
||||
import re
|
||||
from parsers.heading_rules import get_heading_engine
|
||||
engine = get_heading_engine()
|
||||
level, _ = engine.detect(text)
|
||||
return level
|
||||
|
||||
text = text.strip()
|
||||
def _reclassify_text_chunks(chunks: List[MinerUChunk]) -> None:
|
||||
"""
|
||||
二次校正:检测被标记为 text 但实际是表格/表单的 chunk
|
||||
|
||||
# 空文本
|
||||
MinerU 解析 Word 文档时,某些带下划线填空项的表单
|
||||
被标记为 text 类型,需要根据内容特征修正为 table。
|
||||
就地修改 chunks 列表中的 chunk_type 字段。
|
||||
|
||||
检测依据(基于实测数据设计):
|
||||
- 连续下划线 ___ (3个以上) —— Word 表单填空项
|
||||
- 冒号后跟下划线 如 "日期:____" —— 键值对式表单
|
||||
需 >= min_indicators 个指标同时命中才校正,避免误判。
|
||||
"""
|
||||
try:
|
||||
from config import FORM_RECLASSIFY_ENABLED
|
||||
if not FORM_RECLASSIFY_ENABLED:
|
||||
return
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from config import FORM_RECLASSIFY_MIN_INDICATORS
|
||||
min_indicators = FORM_RECLASSIFY_MIN_INDICATORS
|
||||
except ImportError:
|
||||
min_indicators = 2
|
||||
|
||||
# 表单特征指标(编译一次,避免循环内重复编译)
|
||||
# 注意:MinerU 输出的下划线是 Markdown 转义格式 \_\_\_,
|
||||
# 需要同时匹配纯下划线 ___ 和转义下划线 \_\_\_
|
||||
_FORM_INDICATORS = [
|
||||
re.compile(r'(?:\\_|_){3,}'), # 连续下划线(3个以上,含转义格式)
|
||||
re.compile(r'[::]\s*(?:\\_|_){2,}'), # 冒号后跟下划线(含转义格式)
|
||||
]
|
||||
|
||||
reclassified = 0
|
||||
for chunk in chunks:
|
||||
if chunk.chunk_type != 'text':
|
||||
continue
|
||||
text = (chunk.content or '').strip()
|
||||
if not text:
|
||||
return 0
|
||||
continue
|
||||
|
||||
# 中文章节标题模式
|
||||
# 第一章、第二章、... -> h1
|
||||
if re.match(r'^第[一二三四五六七八九十百千万]+[章节篇部]', text):
|
||||
return 1
|
||||
indicator_count = sum(1 for p in _FORM_INDICATORS if p.search(text))
|
||||
if indicator_count >= min_indicators:
|
||||
chunk.chunk_type = 'table'
|
||||
reclassified += 1
|
||||
logger.info(f"表单检测: text -> table, content='{text[:80]}'")
|
||||
|
||||
# 第一条、第二条、... -> h2 (条文编号)
|
||||
if re.match(r'^第[一二三四五六七八九十百千万]+[条款]', text):
|
||||
return 2
|
||||
if reclassified > 0:
|
||||
logger.info(f"表单类型二次校正: 共 {reclassified} 个 text -> table")
|
||||
|
||||
# 数字章节: 1. 2. 3. 或 1、2、3、
|
||||
# 一级标题: 1. 2. 3. (单数字)
|
||||
if re.match(r'^\d+[\.、\s]', text):
|
||||
# 短文本可能是标题
|
||||
if len(text) < 50:
|
||||
return 1
|
||||
|
||||
# 二级标题: 1.1 1.2 2.1 等
|
||||
if re.match(r'^\d+\.\d+[\.、\s]', text):
|
||||
if len(text) < 80:
|
||||
return 2
|
||||
def _validate_category_section_paths(chunks: List[MinerUChunk]) -> None:
|
||||
"""
|
||||
验证分类标题是否被规则引擎正确识别(安全网)
|
||||
|
||||
# 三级标题: 1.1.1 1.1.2 等
|
||||
if re.match(r'^\d+\.\d+\.\d+[\.、\s]', text):
|
||||
if len(text) < 100:
|
||||
return 3
|
||||
当规则引擎正确识别分类标题后,section_stack 自然会更新,
|
||||
不再需要后处理修改 section_path。此函数仅做验证和告警,
|
||||
便于发现规则引擎的遗漏。
|
||||
|
||||
# 英文章节标题
|
||||
# Chapter 1, Section 2, etc.
|
||||
if re.match(r'^(Chapter|Section|Part|Chapter\s+\d+|Section\s+\d+)', text, re.IGNORECASE):
|
||||
return 1
|
||||
支持的模式:A1类:、B2类:、C1类:等(含可选 ** 粗体标记)。
|
||||
"""
|
||||
cat_pattern = re.compile(r'^\*{0,2}[A-Z]\d+[类類]\*{0,2}[::]')
|
||||
|
||||
# 短文本 + 加粗标记 (**xxx**) 可能是标题
|
||||
if re.match(r'^\*\*.+\*\*$', text) and len(text) < 50:
|
||||
return 2
|
||||
missed_count = 0
|
||||
for chunk in chunks:
|
||||
if chunk.chunk_type == 'text':
|
||||
text = (chunk.content or '').strip()
|
||||
if cat_pattern.match(text) and chunk.text_level == 0:
|
||||
missed_count += 1
|
||||
logger.warning(
|
||||
f"分类标题未被识别为标题: '{text[:50]}', "
|
||||
f"section_path='{chunk.section_path}'"
|
||||
)
|
||||
|
||||
# 非常短的文本 (< 20 字符) 可能是标题
|
||||
# 但需要排除常见的非标题短文本
|
||||
if len(text) < 20 and not re.match(r'^[\d\s\.,;:!?,。;:!?、]+$', text):
|
||||
# 排除纯数字、纯标点
|
||||
if re.search(r'[\u4e00-\u9fff]', text): # 包含中文
|
||||
return 2
|
||||
|
||||
return 0
|
||||
if missed_count > 0:
|
||||
logger.warning(
|
||||
f"发现 {missed_count} 个分类标题未被规则引擎识别,"
|
||||
f"请检查 heading_rules 配置"
|
||||
)
|
||||
|
||||
|
||||
def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]:
|
||||
@@ -803,16 +1001,45 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]:
|
||||
else:
|
||||
raise RuntimeError(f"MinerU 输出目录不存在: {auto_dir} 或 {office_dir}")
|
||||
|
||||
# 读取 content_list.json
|
||||
content_list_path = output_subdir / f"{doc_name}_content_list.json"
|
||||
if not content_list_path.exists():
|
||||
content_list_path = output_subdir / f"{doc_name}_content_list_v2.json"
|
||||
# 读取 content_list(v2 含 style 信息,优先使用)
|
||||
try:
|
||||
from config import MINERU_PREFER_V2
|
||||
except ImportError:
|
||||
MINERU_PREFER_V2 = True
|
||||
|
||||
v1_path = output_subdir / f"{doc_name}_content_list.json"
|
||||
v2_path = output_subdir / f"{doc_name}_content_list_v2.json"
|
||||
|
||||
content_list_path = None
|
||||
is_v2 = False
|
||||
|
||||
if MINERU_PREFER_V2:
|
||||
# 优先使用 v2 格式
|
||||
if v2_path.exists():
|
||||
content_list_path = v2_path
|
||||
is_v2 = True
|
||||
elif v1_path.exists():
|
||||
content_list_path = v1_path
|
||||
else:
|
||||
# 优先使用 v1 格式
|
||||
if v1_path.exists():
|
||||
content_list_path = v1_path
|
||||
elif v2_path.exists():
|
||||
content_list_path = v2_path
|
||||
is_v2 = True
|
||||
|
||||
content_list = []
|
||||
if content_list_path.exists():
|
||||
if content_list_path and content_list_path.exists():
|
||||
with open(content_list_path, 'r', encoding='utf-8') as f:
|
||||
content_list = json.load(f)
|
||||
|
||||
# v2 格式需要转换为 v1 兼容的扁平列表
|
||||
if is_v2 and isinstance(content_list, list) and content_list and isinstance(content_list[0], list):
|
||||
content_list = _parse_v2_content_list(content_list)
|
||||
logger.info(f"v2 格式已转换为扁平列表,共 {len(content_list)} 项")
|
||||
|
||||
logger.info(f"读取 content_list: {len(content_list)} 项, 格式={'v2' if is_v2 else 'v1'}")
|
||||
|
||||
# 读取 Markdown
|
||||
md_path = output_subdir / f"{doc_name}.md"
|
||||
markdown_content = ""
|
||||
@@ -857,10 +1084,14 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]:
|
||||
|
||||
if item_type == "text":
|
||||
text = item.get("text", "")
|
||||
v2_styles = item.get("_v2_styles", [])
|
||||
|
||||
# 启发式标题识别(当 text_level 为 0 时)
|
||||
if text_level == 0:
|
||||
text_level = _detect_heading_level(text)
|
||||
from parsers.heading_rules import get_heading_engine
|
||||
engine = get_heading_engine()
|
||||
detected_level, _ = engine.detect(text, style=v2_styles)
|
||||
text_level = detected_level
|
||||
|
||||
# 处理标题
|
||||
title = ""
|
||||
@@ -909,8 +1140,8 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]:
|
||||
else:
|
||||
md_table = ""
|
||||
|
||||
# 提取表格中的嵌入图片
|
||||
table_images = extract_images_from_markdown(md_table) if md_table else []
|
||||
# 从原始 HTML 提取嵌入图片(md_table 经 get_text 转换后已丢失 <img> 标签)
|
||||
table_images = extract_images_from_markdown(table_body) if table_body else []
|
||||
|
||||
chunk = MinerUChunk(
|
||||
content=table_caption or "表格",
|
||||
@@ -1003,8 +1234,14 @@ def _parse_mineru_output(file_path: Path, output_dir) -> Dict[str, Any]:
|
||||
min_merge = 100
|
||||
max_size = 1200
|
||||
|
||||
# 表单类型二次校正(在 _post_process_chunks 之前,因为 table 不参与合并)
|
||||
_reclassify_text_chunks(chunks)
|
||||
|
||||
chunks = _post_process_chunks(chunks, min_merge_size=min_merge, max_chunk_size=max_size)
|
||||
|
||||
# 验证分类标题是否被规则引擎正确识别(安全网,仅告警不修改)
|
||||
_validate_category_section_paths(chunks)
|
||||
|
||||
return {
|
||||
'markdown': "\n".join(markdown_parts),
|
||||
'chunks': chunks,
|
||||
@@ -1349,7 +1586,17 @@ def html_table_to_markdown(html_table: str) -> str:
|
||||
del rowspan_tracker[col_idx]
|
||||
col_idx += 1
|
||||
|
||||
# 提取单元格内容
|
||||
# 提取单元格内容(保留图片引用信息)
|
||||
img_tags = cell.find_all('img')
|
||||
if img_tags:
|
||||
# 单元格包含图片,生成占位标记供 LLM 感知
|
||||
text_part = cell.get_text(strip=True)
|
||||
img_count = len(img_tags)
|
||||
if text_part:
|
||||
content = f"{text_part} [{'图片' if img_count == 1 else f'{img_count}张图片'}]"
|
||||
else:
|
||||
content = f"[{'图片' if img_count == 1 else f'{img_count}张图片'}]"
|
||||
else:
|
||||
content = cell.get_text(strip=True)
|
||||
|
||||
# 处理 rowspan
|
||||
@@ -1630,6 +1877,21 @@ def parse_with_mineru_persistent(
|
||||
chunk.image_path = new_name
|
||||
break
|
||||
|
||||
# 更新表格嵌入图片的路径映射(images 字段)
|
||||
if hasattr(chunk, 'images') and chunk.images:
|
||||
for img_info in chunk.images:
|
||||
if isinstance(img_info, dict) and 'id' in img_info:
|
||||
old_id = img_info['id']
|
||||
if old_id in image_path_map:
|
||||
img_info['id'] = image_path_map[old_id]
|
||||
else:
|
||||
# 兼容:尝试用文件名匹配映射
|
||||
old_basename = os.path.basename(old_id)
|
||||
for old_path, new_name in image_path_map.items():
|
||||
if old_basename == os.path.basename(old_path):
|
||||
img_info['id'] = new_name
|
||||
break
|
||||
|
||||
# 更新结果中的图片路径列表(供外部使用)
|
||||
result['images'] = list(image_path_map.values())
|
||||
|
||||
|
||||
@@ -616,7 +616,7 @@ class FeedbackService:
|
||||
prompt=prompt,
|
||||
model=self.model,
|
||||
temperature=0.7,
|
||||
max_tokens=200
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
if not response:
|
||||
|
||||
@@ -159,7 +159,7 @@ class SessionManager:
|
||||
SELECT id, role, content, metadata, created_at
|
||||
FROM messages
|
||||
WHERE session_id = ?
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT ?
|
||||
''', (session_id, limit))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user