From 5ccfdaf9d104ec8073812429c2d3caec112b3a7b Mon Sep 17 00:00:00 2001 From: lacerate551 <128470311+lacerate551@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:51:11 +0800 Subject: [PATCH] =?UTF-8?q?feat(exam=5Fpkg):=20=E4=BC=98=E5=8C=96=E5=87=BA?= =?UTF-8?q?=E9=A2=98=E7=B3=BB=E7=BB=9F=E7=A8=B3=E5=AE=9A=E6=80=A7=E4=B8=8E?= =?UTF-8?q?=E6=8E=A8=E7=90=86=E6=A8=A1=E5=9E=8B=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - core/llm_utils: MiMo模型自动注入thinking=disabled参数，全局生效 - config: 新增LLM_DISABLE_THINKING配置项（默认true） - generator: 推理模型自适应max_tokens（1.5x）、429限流重试+指数退避、v2管线补题机制 - generator: analyze_document_for_exam新增max_total参数控制AI出题上限 - generator: validate_questions_schema兼容type和question_type字段 - grader: 主观题max_tokens从1000提升至2000、fuzzy_match增加编辑距离容错 - manager: results变量初始化防NameError、透传max_total参数 - api: /exam/generate-smart支持max_total请求参数 --- core/llm_utils.py | 32 +++++++ docs/出题系统测试报告.md | 190 +++++++++++++++++++++++++++++++++++++ exam_pkg/api.py | 14 ++- exam_pkg/generator.py | 200 ++++++++++++++++++++++++++++----------- exam_pkg/grader.py | 84 ++++++++++++++-- exam_pkg/manager.py | 8 +- 6 files changed, 462 insertions(+), 66 deletions(-) create mode 100644 docs/出题系统测试报告.md diff --git a/core/llm_utils.py b/core/llm_utils.py index 72a66b8..2913070 100644 --- a/core/llm_utils.py +++ b/core/llm_utils.py @@ -11,6 +11,24 @@ from typing import List, Optional, Union, Iterator, Callable logger = logging.getLogger(__name__) +# MiMo 推理模型关键词（用于识别需要 thinking 参数的模型） +_MIMO_MODEL_KEYWORDS = ('mimo',) + +def _is_mimo_model(model_name: str) -> bool: + """判断是否为小米 MiMo 模型（支持 thinking 参数）""" + if not model_name: + return False + return any(kw in model_name.lower() for kw in _MIMO_MODEL_KEYWORDS) + +def _inject_mimo_thinking(kwargs: dict, model: str, disable_thinking: bool = True) -> dict: + """为 MiMo 模型注入 thinking 参数。返回更新后的 kwargs。""" + if not _is_mimo_model(model): + return kwargs + extra = dict(kwargs.get('extra_body') or {}) + extra['thinking'] = {'type': 'disabled' if disable_thinking else 'enabled'} + kwargs['extra_body'] = extra + return kwargs + def call_llm( client, @@ -57,6 +75,13 @@ def call_llm( if messages is None: messages = [{"role": "user", "content": prompt}] + # MiMo 模型自动注入 thinking 参数 + try: + from config import LLM_DISABLE_THINKING + except ImportError: + LLM_DISABLE_THINKING = True + kwargs = _inject_mimo_thinking(kwargs, model, disable_thinking=LLM_DISABLE_THINKING) + try: response = client.chat.completions.create( model=model, @@ -145,6 +170,13 @@ def call_llm_stream( if messages is None: messages = [{"role": "user", "content": prompt}] + # MiMo 模型自动注入 thinking 参数 + try: + from config import LLM_DISABLE_THINKING + except ImportError: + LLM_DISABLE_THINKING = True + kwargs = _inject_mimo_thinking(kwargs, model, disable_thinking=LLM_DISABLE_THINKING) + try: stream = client.chat.completions.create( model=model, diff --git a/docs/出题系统测试报告.md b/docs/出题系统测试报告.md new file mode 100644 index 0000000..48be1f9 --- /dev/null +++ b/docs/出题系统测试报告.md @@ -0,0 +1,190 @@ +# 出题系统测试报告 + +**测试日期**: 2026-06-22 +**测试环境**: mimo-v2.5（关闭推理模式）、2.docx（文明吸烟环境建设标准） +**测试方式**: curl 接口测试 + 代码审查 + +--- + +## 一、测试总览 + +| # | 测试项 | 接口 | 结果 | 耗时 | +|---|--------|------|------|------| +| 1 | 参数出题 | POST /exam/generate | ✅ 成功 | ~87s (4题) | +| 2 | AI一键出题 | POST /exam/generate-smart | ⚠️ 成功但有上限失控 | ~244s (50题) | +| 3 | 偏门题型(纯主观) | POST /exam/generate | ✅ 成功 | ~87s (5题) | +| 4 | 偏门题型(多选+填空) | POST /exam/generate | ✅ 成功 | ~60s (6题) | +| 5 | 跨调用去重 | POST /exam/generate + exclude_stems | ✅ 去重有效 | ~60s (5题) | +| 6 | 批题(4题型混合) | POST /exam/grade | ✅ 全部成功 | ~30s | + +--- + +## 二、测试详情 + +### Test 1: 参数出题 (single_choice:2, fill_blank:1, true_false:1) +- **结果**: 4题全部生成成功，题型匹配 +- **问题**: + - ❌ 题目多样性不足：4题中3题考同一个知识点（"标准的解释机构"） + - ❌ source_trace 数据缺失（chunk_id=?, page=?） + +### Test 2: AI一键出题 (generate-smart) +- **AI分析**: 检测到54个知识点，推荐50道题（20单选+15判断+10多选+5填空） +- **结果**: 50题全部生成，12个不同章节，0题干重复 +- **问题**: + - ❌ **P0 上限失控**: prompt 要求"不超过20"，但AI返回50，代码没有 enforce 上限校验 + - ⚠️ 50题意味着20-30次LLM调用，极易触发429限流 + +### Test 3: 纯主观题 (subjective:5) +- **结果**: 5题全部生成，覆盖不同章节，scoring_points 完整 +- **耗时**: 87秒（关闭推理模式后正常） + +### Test 4: 多选+填空 (multiple_choice:3, fill_blank:3) +- **结果**: 6题全部生成 +- **问题**: + - ❌ **填空题答案格式不一致**: `content.answer` 是 `[["消费水平较高"]]` (list of list)，但没有 `data.reference_answer` 字段，只有 `data.blank_count` + - ⚠️ 与 grader 期望的格式可能不匹配 + +### Test 5: 跨调用去重 (exclude_stems) +- **结果**: 3个排除题干全部未出现，去重有效 ✅ + +### Test 6: 批题 (grade) +- **单选题答错**: score=0, correct=false ✅ +- **判断题答对**: score=2, correct=true ✅ +- **填空题部分对**: score=4.0 (满分4), blank_scores=[4.0] ✅ +- **主观题(低质量回答)**: score=2.0/10, 4个scoring_point逐项评分 ✅ +- **总评**: 得分率44.4%，评分合理 + +--- + +## 三、发现的问题（按严重度排序） + +### P0 - 严重问题 + +#### 1. 🚨 AI一键出题上限失控 (generator.py:1053-1068) +**位置**: `analyze_document_for_exam()` 函数 +**现象**: prompt 写了"所有数量之和不要超过 min(total_knowledge_points * 2, 20)"，但 LLM 返回50题，代码直接采纳 +**根因**: 代码只做了"题型合法性"校验，没有对总数做上限 enforce +```python +# 当前代码 (generator.py:1053-1059) - 只校验单题型合法性，无总数限制 +valid_types = ['single_choice', 'multiple_choice', 'true_false', 'fill_blank', 'subjective'] +question_types = {} +for q_type in valid_types: + count = result.get('question_types', {}).get(q_type, 0) + if isinstance(count, int) and count >= 0: + question_types[q_type] = count # 直接采纳，无上限 +``` +**建议**: 增加总数上限校验，超过 `max_questions`（建议20）时按比例缩减 +```python +total = sum(question_types.values()) +max_questions = 20 +if total > max_questions: + ratio = max_questions / total + question_types = {k: max(0, round(v * ratio)) for k, v in question_types.items()} +``` + +#### 2. 🚨 LLM 调用无限流机制，429 风暴 (generator.py:290-303, 540-543) +**位置**: `generate_questions_structured()` 主循环 + `_generate_with_retry()` 重试 +**现象**: 并发出题时大量429错误，重试退避太短(1s/2s/4s)，3次全败后放弃该知识点 +**根因**: +- 主循环 for 逐知识点调用 LLM，无请求间隔（补题函数有 sleep(1)，主循环没有） +- 429 重试退避 `2^attempt` 秒 (1/2/4s) 对 API 限流不够 +- 无全局限流器（令牌桶/漏桶） +**建议**: +1. 主循环每次 LLM 调用后加 `time.sleep(1.5)` 最小间隔 +2. 429 退避改为指数+抖动: `min(30, 2 ** attempt + random.uniform(0, 2))` +3. 长期: 引入 `tenacity` 或自实现令牌桶限流器 + +#### 3. 🚨 validate_questions_schema 只认 `type` 不认 `question_type` (generator.py:154) +**位置**: `validate_questions_schema()` 函数 +**现象**: LLM 返回的题目可能用 `question_type` 字段，但校验只检查 `q.get('type')`，导致有效题目被丢弃 +**对比**: `_validate_question_types()` (第448行) 做了兼容: `q_type = q.get('question_type') or q.get('type')` +```python +# 当前代码 (generator.py:154) - 不兼容 +if q.get('type') not in VALID_TYPES: + continue # question_type 字段的题目被丢弃！ + +# 应改为 +q_type = q.get('question_type') or q.get('type') +if q_type not in VALID_TYPES: + continue +``` + +### P1 - 中等问题 + +#### 4. ⚠️ 填空题答案格式不一致 +**现象**: 出题返回 `content.answer = [["答案1"], ["答案2"]]` (list of list)，但无 `data.reference_answer` +**影响**: 前端/本地数据库可能期望 `reference_answer` 字段 +**建议**: 统一在 `data` 中增加 `reference_answer` 字段，与 `content.answer` 保持一致 + +#### 5. ⚠️ grader max_tokens 对推理模型不足 (grader.py:438) +**位置**: `_grade_subjective()` 方法 +**现象**: `max_tokens=_get_effective_max_tokens(1000, self.model)` +- 推理模型关闭推理时: 1000 tokens 勉强够 +- 推理模型开启推理时: 思考链消耗 800+ tokens，content 为空 +**当前状态**: 关闭推理模式后本次测试通过，但**开启推理模式会再次失败** +**建议**: 基础 max_tokens 从 1000 提升至 2000 + +#### 6. ⚠️ local_db._detect_question_type 使用旧格式 (local_db.py:378-383) +**位置**: `_detect_question_type()` 方法 +**现象**: 检测 `options` 和 `reference_answer` 字段来判断题型，但新格式用 `content.data.options` 和 `content.answer` +```python +# 当前代码 - 检查顶层 options +if 'options' in question and question['options']: + return 'choice' +elif 'reference_answer' in question: + return 'short_answer' + +# 应改为检查 content 内部结构 +content = question.get('content', {}) +data = content.get('data', {}) +if data.get('options'): + return 'choice' +elif question.get('question_type') in ('fill_blank', 'subjective', ...): + return question['question_type'] +``` + +#### 7. ⚠️ 题目多样性不足 +**现象**: Test 1 中 4 题有 3 题考同一知识点（"标准的解释机构"） +**根因**: `_assign_questions_to_kps()` 可能将多个题型分配给同一知识点 +**建议**: 增加"同一知识点最多出 N 道题"的限制（建议 N=2） + +### P2 - 低优先级 + +#### 8. 💡 source_trace 数据偶尔缺失 +**现象**: Test 1 中部分题目的 chunk_id 和 page 显示为 `?` +**可能原因**: `find_referenced_chunks` 匹配失败时的 fallback 显示 + +#### 9. 💡 多选题答案格式 +**现象**: 多选题 answer 为 `['A', 'B']` 或 `['A', 'B', 'D']` (list)，前端需处理 +**建议**: 在 API 文档中明确多选题 answer 格式为 list + +#### 10. 💡 补题机制未与已有题目交叉去重 +**现象**: 补题时只检查本次生成的题干，不检查已有题目 +**建议**: 补题函数接受 `exclude_stems` 参数 + +--- + +## 四、测试通过项 ✅ + +1. **基础出题功能**: 5种题型均可正常生成 +2. **AI智能分析**: 文档分析→题型推荐→出题流程完整 +3. **补题机制**: 知识点出题失败后自动补题 +4. **跨调用去重**: exclude_stems 功能正常，排除的题干不再出现 +5. **批题功能**: 客观题精确匹配，填空题部分给分，主观题LLM逐项评分 +6. **source_trace**: 大部分题目有完整的溯源信息（chunk_id, section, snippet） +7. **JSON解析**: mimo-v2.5 返回的 JSON 格式（含 markdown 包裹）可正常解析 +8. **题目格式**: 单选题 options 为 list of dict `[{key, content}]`，格式规范 + +--- + +## 五、建议修复优先级 + +| 优先级 | 问题 | 修复难度 | 影响范围 | +|--------|------|----------|----------| +| P0 | AI出题上限失控 | 简单 | smart 出题 | +| P0 | LLM 429 限流 | 中等 | 全部出题 | +| P0 | validate_questions_schema type 字段 | 简单 | 全部出题 | +| P1 | 填空题答案格式统一 | 简单 | 填空题+批题 | +| P1 | grader max_tokens | 简单 | 主观题批题 | +| P1 | local_db 旧格式 | 中等 | 本地存储 | +| P1 | 题目多样性控制 | 中等 | 出题质量 | diff --git a/exam_pkg/api.py b/exam_pkg/api.py index 9296544..d05f3e9 100644 --- a/exam_pkg/api.py +++ b/exam_pkg/api.py @@ -210,6 +210,7 @@ def api_generate_smart(): "file_path": "public/产品手册.pdf", "collection": "public_kb", "difficulty": 3, // 可选，默认 3 + "max_total": 20, // 可选，AI出题总数上限。不传则不限制 "options": {} // 可选 } @@ -231,6 +232,16 @@ def api_generate_smart(): if diff_error: return error_response("INVALID_PARAMS", BAD_REQUEST, diff_error, http_status=400) + # 可选：AI出题总数上限（不传则不限制） + max_total = data.get('max_total') + if max_total is not None: + try: + max_total = int(max_total) + if max_total <= 0: + return error_response("INVALID_PARAMS", BAD_REQUEST, "max_total 必须为正整数", http_status=400) + except (ValueError, TypeError): + return error_response("INVALID_PARAMS", BAD_REQUEST, "max_total 必须为整数", http_status=400) + # 校验排除题干列表（可选） exclude_stems = data.get('exclude_stems') stems_error = validate_exclude_stems(exclude_stems) @@ -255,7 +266,8 @@ def api_generate_smart(): from exam_pkg.manager import analyze_file_for_exam ai_analysis = analyze_file_for_exam( file_path=file_path, - collection=collection + collection=collection, + max_total=max_total ) question_types = ai_analysis.get('question_types', {}) diff --git a/exam_pkg/generator.py b/exam_pkg/generator.py index 213c557..1bfceda 100644 --- a/exam_pkg/generator.py +++ b/exam_pkg/generator.py @@ -26,19 +26,38 @@ import logging logger = logging.getLogger(__name__) # 导入 LLM 工具函数 -from core.llm_utils import call_llm, parse_json_list_from_response +from core.llm_utils import call_llm, parse_json_list_from_response, extract_json_list, extract_json_object # 导入 LLM 配置 try: - from config import API_KEY, BASE_URL, MODEL + from config import API_KEY, BASE_URL, MODEL, LLM_TEMPERATURE, LLM_MAX_TOKENS LLM_AVAILABLE = True except ImportError: API_KEY = None BASE_URL = None MODEL = None + LLM_TEMPERATURE = 0.7 + LLM_MAX_TOKENS = 4000 LLM_AVAILABLE = False +# 推理模型识别：这些模型会消耗额外 token 用于思考链，需要更大的 max_tokens 预算 +_REASONING_MODEL_KEYWORDS = ('mimo', 'qwq', 'deepseek-r1', 'deepseek-reasoner', 'o1', 'o3') + +def _is_reasoning_model(model_name: str) -> bool: + """判断是否为推理模型（需要额外思考链 token 预算）""" + if not model_name: + return False + name_lower = model_name.lower() + return any(kw in name_lower for kw in _REASONING_MODEL_KEYWORDS) + +def _get_effective_max_tokens(base_max: int, model_name: str) -> int: + """根据模型类型计算实际 max_tokens。推理模型需要 1.5x 预算给思考链""" + if _is_reasoning_model(model_name): + return max(base_max, int(base_max * 1.5)) + return base_max + + # ==================== 辅助函数 ==================== def group_chunks_by_section(chunks: List[Dict]) -> Dict[str, List[Dict]]: @@ -131,8 +150,9 @@ def validate_questions_schema(questions: List[Dict]) -> List[Dict]: validated = [] for q in questions: - # 必须有 type - if q.get('type') not in VALID_TYPES: + # 必须有 type 或 question_type + q_type = q.get('question_type') or q.get('type') + if q_type not in VALID_TYPES: continue # 必须有 content @@ -145,7 +165,7 @@ def validate_questions_schema(questions: List[Dict]) -> List[Dict]: continue # 选项题必须有 options - if q['type'] in ['single_choice', 'multiple_choice']: + if q_type in ['single_choice', 'multiple_choice']: if not content.get('data', {}).get('options'): continue @@ -359,8 +379,8 @@ class QuestionGenerator: client=self.client, prompt=prompt, model=self.model, - temperature=0.7, - max_tokens=4000, + temperature=LLM_TEMPERATURE, + max_tokens=_get_effective_max_tokens(4000, self.model), messages=messages ) if not content: @@ -493,25 +513,37 @@ class QuestionGenerator: return type_names.get(q_type, q_type) def _call_llm(self, prompt: str) -> str: - """调用本地 LLM（OpenAI 兼容接口）""" + """调用本地 LLM（OpenAI 兼容接口），支持 429 限流重试""" if not self.client: raise ValueError("LLM 客户端未初始化，请检查 config.py 中的 API_KEY 配置") + import time + messages = [ {"role": "system", "content": "你是一个专业的出题专家，擅长根据文档内容生成各类考试题目。你必须严格按照JSON格式输出，不要有任何其他内容。"}, {"role": "user", "content": prompt} ] - result = call_llm( - client=self.client, - prompt=prompt, - model=self.model, - temperature=0.7, - max_tokens=4000, - messages=messages - ) - if result is None: - raise Exception("LLM 调用失败") - return result + + # 重试机制：429 限流时指数退避（最多重试 3 次） + for attempt in range(4): + result = call_llm( + client=self.client, + prompt=prompt, + model=self.model, + temperature=LLM_TEMPERATURE, + max_tokens=_get_effective_max_tokens(4000, self.model), + messages=messages + ) + if result is not None: + return result + + # 可能是 429 限流，等待后重试 + if attempt < 3: + wait_time = 2 ** attempt # 1s, 2s, 4s + logger.warning(f" LLM 调用失败，{wait_time}s 后重试 ({attempt+1}/3)...") + time.sleep(wait_time) + + raise Exception("LLM 调用失败（已重试 3 次）") def _get_format_examples(self) -> str: """返回各题型格式示例（覆盖全部 5 种题型）""" @@ -610,7 +642,6 @@ class QuestionGenerator: ) # 清理纯标点符号 - import re all_content = re.sub(r'^[\s\*\-\d\.。、，：:；;]+$', '', all_content, flags=re.MULTILINE) all_content = all_content.strip() @@ -645,27 +676,24 @@ class QuestionGenerator: 请直接输出 JSON 数组：""" - try: - response = self._call_llm(prompt) + for _attempt in range(2): + try: + response = self._call_llm(prompt) + if not response or not response.strip(): + continue - # 清理响应（移除可能的 markdown 标记） - response = response.strip() - if response.startswith('```'): - lines = response.split('\n') - response = '\n'.join(lines[1:-1] if lines[-1] == '```' else lines[1:]) - - # 解析 JSON - result = json.loads(response) - if isinstance(result, list): - return [ - {"name": kp, "section": section} - for kp in result[:max_points] - if isinstance(kp, str) and 3 <= len(kp) <= 30 - ] - except json.JSONDecodeError as e: - logger.error(f" 知识点 JSON 解析失败: {e}") - except Exception as e: - logger.error(f" 知识点提取失败: {e}") + # 使用多策略 JSON 提取（支持 markdown 代码块、正则回退） + result = extract_json_list(response) + if isinstance(result, list) and result: + return [ + {"name": kp, "section": section} + for kp in result[:max_points] + if isinstance(kp, str) and 3 <= len(kp) <= 30 + ] + # 解析结果无效，重试一次 + logger.warning(f" 知识点提取返回无效结果(尝试 {_attempt+1}/2)，重试中...") + except Exception as e: + logger.error(f" 知识点提取失败(尝试 {_attempt+1}/2): {e}") return [] @@ -918,7 +946,7 @@ def generate_questions_from_content( return generator.generate_questions_structured(chunks, document_name, question_types, difficulty) -def analyze_document_for_exam(chunks: List[Dict]) -> Dict[str, Any]: +def analyze_document_for_exam(chunks: List[Dict], max_total: int = None) -> Dict[str, Any]: """ AI 智能分析文档内容，决定适合的题型和数量 @@ -1026,14 +1054,14 @@ def analyze_document_for_exam(chunks: List[Dict]) -> Dict[str, Any]: try: response = generator._call_llm(prompt) - response = response.strip() + if not response or not response.strip(): + return _generate_default_question_types(total_knowledge_points) - # 清理 markdown 代码块 - if response.startswith('```'): - lines = response.split('\n') - response = '\n'.join(lines[1:-1] if lines[-1] == '```' else lines[1:]) - - result = json.loads(response) + # 使用多策略 JSON 提取（支持 markdown 代码块、正则回退） + result = extract_json_object(response) + if not isinstance(result, dict): + logger.error(f"AI 分析返回非对象类型: {type(result)}") + return _generate_default_question_types(total_knowledge_points) # 验证和清理结果 valid_types = ['single_choice', 'multiple_choice', 'true_false', 'fill_blank', 'subjective'] @@ -1046,6 +1074,24 @@ def analyze_document_for_exam(chunks: List[Dict]) -> Dict[str, Any]: # 过滤不适合的题型 suitable_types = [t for t, c in question_types.items() if c > 0] + # 总数上限校验：如果指定了 max_total，超过时按比例缩减 + if max_total and max_total > 0: + total = sum(question_types.values()) + if total > max_total: + ratio = max_total / total + question_types = {k: max(0, round(v * ratio)) for k, v in question_types.items()} + # 修正四舍五入误差 + diff = max_total - sum(question_types.values()) + if diff > 0: + # 把差额分配给最大的题型 + for k in sorted(question_types, key=question_types.get, reverse=True): + question_types[k] += 1 + diff -= 1 + if diff <= 0: + break + logger.info(f" AI 推荐 {total} 题，按上限 {max_total} 缩减为 {sum(question_types.values())} 题") + suitable_types = [t for t, c in question_types.items() if c > 0] + return { "total_knowledge_points": result.get('total_knowledge_points', total_knowledge_points), "suitable_types": suitable_types, @@ -1053,10 +1099,6 @@ def analyze_document_for_exam(chunks: List[Dict]) -> Dict[str, Any]: "reason": result.get('reason', 'AI 分析完成') } - except json.JSONDecodeError as e: - logger.error(f"AI 分析结果 JSON 解析失败: {e}") - # 降级：根据知识点数量生成默认配置 - return _generate_default_question_types(total_knowledge_points) except Exception as e: logger.error(f"AI 分析失败: {e}") return _generate_default_question_types(total_knowledge_points) @@ -1156,6 +1198,10 @@ def generate_questions_structured_v2( # 提取知识点 kps = generator._extract_knowledge_points(section, section_chunks, max_points=3) + # API 限流保护：连续 LLM 调用间隔 1 秒 + import time + time.sleep(1) + # 全局去重 for kp in kps: kp_name = kp['name'] @@ -1219,6 +1265,10 @@ def generate_questions_structured_v2( prompt, {q_type: 1}, kp_chunks, document_name, max_retries=2 ) + # API 限流保护：连续 LLM 调用间隔 1 秒 + import time + time.sleep(1) + if success and questions: all_questions.extend(questions) else: @@ -1241,6 +1291,28 @@ def generate_questions_structured_v2( type_counts[q.get('question_type')] += 1 logger.info(f" 题型分布: {dict(type_counts)}") + # 4.4 补题：如果某题型数量不足，使用 chunks 补充 + shortage_types = {} + for q_type, target_count in question_types.items(): + actual_count = type_counts.get(q_type, 0) + if actual_count < target_count: + shortage_types[q_type] = target_count - actual_count + + if shortage_types: + logger.info(f" [v2] 补题: 缺少题型 {dict(shortage_types)}") + for q_type, shortage in shortage_types.items(): + logger.info(f" 补充 {q_type} {shortage} 道...") + extra = generator._makeup_questions(chunks, q_type, shortage, difficulty, document_name) + # 补题也需去重 + extra_deduped = _deduplicate_questions(extra, exclude_stems=exclude_stems) + final.extend(extra_deduped[:shortage]) + + # 补题后重新统计 + type_counts = defaultdict(int) + for q in final: + type_counts[q.get('question_type')] += 1 + logger.info(f" 补题后题型分布: {dict(type_counts)}") + return final @@ -1404,9 +1476,27 @@ def _deduplicate_questions(questions: List[Dict], exclude_stems: List[str] = Non # 预填已有题目的题干前缀，使新生成的题目与已有题目冲突时被过滤 if exclude_stems: + _valid_types = ('single_choice', 'multiple_choice', 'true_false', 'fill_blank', 'subjective') for stem in exclude_stems: seen_stems.add(stem[:80]) - seen_kp_type.add(f"{stem[:30]}_") # 通配题型匹配 + # 用 exclude_stem 自身长度做前缀键（排除题干通常短于30字） + # 新题目的 stem[:30] 如果以此前缀开头，[:len(prefix)] 后就能匹配 + _prefix = stem[:30] + for _qt in _valid_types: + seen_kp_type.add(f"{_prefix}_{_qt}") + + # 辅助函数：检查新题干是否匹配任何 exclude 前缀 + _exclude_prefixes = [] + if exclude_stems: + _exclude_prefixes = [s[:30] for s in exclude_stems] + + def _matches_exclude(stem_text: str) -> bool: + """检查题干前30字是否以某个 exclude 前缀开头""" + _stem30 = stem_text[:30] + for _ep in _exclude_prefixes: + if _stem30.startswith(_ep): + return True + return False deduped = [] @@ -1419,6 +1509,10 @@ def _deduplicate_questions(questions: List[Dict], exclude_stems: List[str] = Non if stem_key in seen_stems: continue + # 跨调用排除：题干前缀匹配到 exclude_stems 则跳过 + if _matches_exclude(stem): + continue + # 知识点 + 题型去重 kp_type_key = f"{stem[:30]}_{q.get('question_type')}" if kp_type_key in seen_kp_type: diff --git a/exam_pkg/grader.py b/exam_pkg/grader.py index c2b6496..acfbbf8 100644 --- a/exam_pkg/grader.py +++ b/exam_pkg/grader.py @@ -37,6 +37,21 @@ except ImportError: MODEL = None LLM_AVAILABLE = False +# 推理模型识别（与 generator.py 共享同一套关键词） +_REASONING_MODEL_KEYWORDS = ('mimo', 'qwq', 'deepseek-r1', 'deepseek-reasoner', 'o1', 'o3') + +def _is_reasoning_model(model_name: str) -> bool: + if not model_name: + return False + name_lower = model_name.lower() + return any(kw in name_lower for kw in _REASONING_MODEL_KEYWORDS) + +def _get_effective_max_tokens(base_max: int, model_name: str) -> int: + """推理模型需要 1.5x token 预算给思考链""" + if _is_reasoning_model(model_name): + return max(base_max, int(base_max * 1.5)) + return base_max + # ==================== 装饰器 ==================== @@ -169,19 +184,56 @@ def grade_fill_blank(answer: Dict) -> Dict: def fuzzy_match(student_answer: str, correct_answer: str) -> bool: """ - 模糊匹配（支持同义词） + 模糊匹配（支持同义词和小编辑距离容错） - 当前实现：精确匹配（忽略前后空格、大小写） - TODO: 可以扩展为语义相似度匹配 + 策略： + 1. 精确匹配（去空格、转小写、统一标点） + 2. 编辑距离容错（≥4字答案允许≤2字符差异） """ if not student_answer or not correct_answer: return False - # 标准化：去空格、转小写 - s = student_answer.strip().lower() - c = correct_answer.strip().lower() + # 标准化：去空格、转小写、统一标点 + def _normalize(text: str) -> str: + t = text.strip().lower() + # 统一常见中文标点变体 + t = t.replace('（', '(').replace('）', ')').replace('，', ',') + t = t.replace('；', ';').replace('：', ':').replace('"', '"').replace('"', '"') + return t - return s == c + s = _normalize(student_answer) + c = _normalize(correct_answer) + + if s == c: + return True + + # 编辑距离容错：答案≥4字时允许≤2字符差异 + if len(s) >= 4 and len(c) >= 4: + dist = _edit_distance(s, c) + if dist <= 2: + return True + + return False + + +def _edit_distance(s1: str, s2: str) -> int: + """计算两个字符串的编辑距离（Levenshtein）""" + if len(s1) < len(s2): + return _edit_distance(s2, s1) + if len(s2) == 0: + return len(s1) + + prev_row = list(range(len(s2) + 1)) + for i, c1 in enumerate(s1): + curr_row = [i + 1] + for j, c2 in enumerate(s2): + # 插入、删除、替换 + insertions = prev_row[j + 1] + 1 + deletions = curr_row[j] + 1 + substitutions = prev_row[j] + (c1 != c2) + curr_row.append(min(insertions, deletions, substitutions)) + prev_row = curr_row + return prev_row[-1] # ==================== AnswerGrader 类 ==================== @@ -237,7 +289,21 @@ class AnswerGrader: # 🔥 P1 改进：并发调用 LLM 批阅主观题 if llm_questions: - self._grade_subjective_concurrently(llm_questions, results_map) + try: + self._grade_subjective_concurrently(llm_questions, results_map) + except Exception as e: + logger.error(f"主观题并发批阅整体异常: {e}") + # 兜底：为所有未完成的主观题设置失败状态 + for ans in llm_questions: + qid = ans.get('question_id') + if qid not in results_map: + results_map[qid] = { + "question_id": qid, + "score": 0, + "max_score": ans.get('max_score', 10), + "grading_status": "failed", + "details": {"error": f"批阅系统异常: {str(e)}"} + } # 🔥 P1 改进：按原始顺序重组结果 results = [results_map.get(ans.get('question_id')) for ans in answers] @@ -369,7 +435,7 @@ class AnswerGrader: prompt=prompt, model=self.model, temperature=0.3, - max_tokens=1000, + max_tokens=_get_effective_max_tokens(2000, self.model), messages=messages ) if result is None: diff --git a/exam_pkg/manager.py b/exam_pkg/manager.py index 7aa9b5f..5a1fcca 100644 --- a/exam_pkg/manager.py +++ b/exam_pkg/manager.py @@ -154,7 +154,8 @@ def generate_questions_from_file( def analyze_file_for_exam( file_path: str, collection: str, - top_k: int = 50 + top_k: int = 50, + max_total: int = None ) -> Dict[str, Any]: """ 分析文件内容，返回 AI 推荐的题型和数量 @@ -193,7 +194,7 @@ def analyze_file_for_exam( } # 2. 调用 AI 分析 - return analyze_document_for_exam(chunks) + return analyze_document_for_exam(chunks, max_total=max_total) def retrieve_file_chunks_for_analysis( @@ -311,6 +312,7 @@ def retrieve_file_chunks( engine = get_engine() # 按优先级遍历 collections，找到文件即停止 + results = None for coll in collections: # 尝试两种格式：文件名和完整路径 for source_filter in [filename, file_path]: @@ -330,7 +332,7 @@ def retrieve_file_chunks( break # 外层循环跳出 chunks = [] - if results.get('documents') and results['documents'][0]: + if results and results.get('documents') and results['documents'][0]: for i, (doc, meta, score) in enumerate(zip( results['documents'][0], results['metadatas'][0],