fix(security): 代码审查安全加固 — 三批次修复(6H/13M/12L)
第一批(快速修复): - H6: main.py --debug 默认值 True→False,防止 Werkzeug RCE - M2+M3: /search 增加 validate_query + top_k 范围限制(1-50) - M4: context_count 范围限制(0-10) + 异常捕获 - L3: assert → raise RuntimeError(生产环境 API Key 检查) - H1: SSE 错误事件移除 traceback 字段 第二批(安全加固): - H2+H3: 文档接口路径遍历 realpath 校验 + 文件类型/大小限制 - H4+H5: 批量上传文件大小检查 - M6: LIKE 查询通配符转义 - M1: 37 处 str(e) 异常信息统一脱敏(6 文件) - M5: CORS 生产环境限制来源 - M7: SESSION_MANAGER None 保护(503) - M11: subprocess 参数注入防护(白名单 + -- 分隔符) 第三批(架构改进): - M8+M9: 提取 JSON 解析共享工具(extract_json_object/list) - M10: Prompt 注入检测防御(prompt_guard.py) - M12: 解析器文件大小限制(Excel 50MB/TXT 20MB/PDF 100MB) - M13: 全局单例竞态条件双重检查锁定(engine/bm25/intent_analyzer)
This commit is contained in:
@@ -24,6 +24,9 @@ logger = logging.getLogger(__name__)
|
||||
# 大表切片阈值
|
||||
MAX_ROWS_PER_CHUNK = 200
|
||||
|
||||
# 文件大小限制
|
||||
MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnifiedChunk:
|
||||
@@ -67,6 +70,13 @@ def parse_excel(
|
||||
}
|
||||
"""
|
||||
filepath = Path(filepath)
|
||||
|
||||
# 检查文件大小
|
||||
if filepath.exists():
|
||||
file_size = filepath.stat().st_size
|
||||
if file_size > MAX_FILE_SIZE:
|
||||
raise ValueError(f"文件过大: {file_size / 1024 / 1024:.1f}MB,最大允许 {MAX_FILE_SIZE / 1024 / 1024:.0f}MB")
|
||||
|
||||
if not filepath.exists():
|
||||
raise FileNotFoundError(f"文件不存在: {filepath}")
|
||||
|
||||
|
||||
@@ -46,6 +46,9 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 文件大小限制
|
||||
MAX_PDF_SIZE = 100 * 1024 * 1024 # 100MB
|
||||
|
||||
# 支持的文件格式
|
||||
SUPPORTED_FORMATS = {
|
||||
'.pdf': 'PDF 文档',
|
||||
@@ -176,6 +179,11 @@ def parse_with_mineru_online(
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
|
||||
# 检查文件大小
|
||||
file_size = file_path.stat().st_size
|
||||
if file_size > MAX_PDF_SIZE:
|
||||
raise ValueError(f"文件过大: {file_size / 1024 / 1024:.1f}MB,最大允许 {MAX_PDF_SIZE / 1024 / 1024:.0f}MB")
|
||||
|
||||
logger.info(f"使用 MinerU 在线 API 解析: {file_path.name}")
|
||||
|
||||
# 读取文件内容
|
||||
@@ -600,6 +608,11 @@ def parse_with_mineru(
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
|
||||
# 检查文件大小
|
||||
file_size = file_path.stat().st_size
|
||||
if file_size > MAX_PDF_SIZE:
|
||||
raise ValueError(f"文件过大: {file_size / 1024 / 1024:.1f}MB,最大允许 {MAX_PDF_SIZE / 1024 / 1024:.0f}MB")
|
||||
|
||||
# 检查文件格式
|
||||
suffix = file_path.suffix.lower()
|
||||
if suffix not in SUPPORTED_FORMATS:
|
||||
@@ -629,8 +642,17 @@ def parse_with_mineru(
|
||||
if not mineru_exe.exists():
|
||||
mineru_exe = "mineru" # 回退到系统 PATH
|
||||
|
||||
# 参数白名单校验,防止注入非法参数
|
||||
ALLOWED_BACKENDS = {'auto', 'pipeline', 'vlm', 'vlm-sglang', 'vlm-auto-engine', 'hybrid-auto-engine', 'ocr'}
|
||||
ALLOWED_LANGS = {'ch', 'en', 'ch_lite', 'en_lite', 'formula', 'table'}
|
||||
if backend not in ALLOWED_BACKENDS:
|
||||
backend = 'pipeline'
|
||||
if lang not in ALLOWED_LANGS:
|
||||
lang = 'ch'
|
||||
|
||||
cmd = [
|
||||
str(mineru_exe),
|
||||
"--",
|
||||
"-p", str(file_path),
|
||||
"-o", str(output_dir),
|
||||
"-m", "auto",
|
||||
@@ -1458,6 +1480,11 @@ def parse_with_mineru_persistent(
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
|
||||
# 检查文件大小
|
||||
file_size = file_path.stat().st_size
|
||||
if file_size > MAX_PDF_SIZE:
|
||||
raise ValueError(f"文件过大: {file_size / 1024 / 1024:.1f}MB,最大允许 {MAX_PDF_SIZE / 1024 / 1024:.0f}MB")
|
||||
|
||||
# 计算文件 hash,用于隔离输出目录
|
||||
file_hash = compute_file_hash(str(file_path))
|
||||
output_dir = Path(output_base) / file_hash
|
||||
|
||||
@@ -5,12 +5,25 @@ TXT 文本解析器
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 文件大小限制
|
||||
MAX_TXT_SIZE = 20 * 1024 * 1024 # 20MB
|
||||
|
||||
|
||||
def extract_text_from_txt(filepath):
|
||||
"""从TXT提取文本"""
|
||||
# 检查文件大小
|
||||
try:
|
||||
file_size = os.path.getsize(filepath)
|
||||
if file_size > MAX_TXT_SIZE:
|
||||
logger.error(f"TXT文件过大: {file_size / 1024 / 1024:.1f}MB")
|
||||
return ""
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
Reference in New Issue
Block a user