sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)

同步内容:
- knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback
- services/feedback.py: FAQ max_tokens 200→512
- services/session.py: 消息排序增加 id DESC 次级排序
- knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块
- knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略
- knowledge/document.py: 文档删除时清理孤儿文件
- api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援)
- core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑
- cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
lacerate551
2026-06-21 14:49:46 +08:00
parent d6d27b70f3
commit 258be54df7
9 changed files with 553 additions and 26 deletions

View File

@@ -293,6 +293,112 @@ def _process_table_doc(doc: str, meta: Dict) -> str:
return doc
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
min_score: float) -> List[Dict]:
"""
BM25-CrossEncoder 分歧检测救援:当 BM25 排名靠前top-3的切片
被 CrossEncoder rerank 压制(分数低于 min_score 或被截断)时,
将其分数提升至保底值,使其通过后续的 min_score 过滤。
适用场景:
- BM25 top-1/top-2 的切片(精确关键词匹配强信号)被 rerank 评分极低
- 正确切片在 rerank 后被截断,根本不在 contexts 中
Args:
contexts: 全部上下文切片
search_result: engine.search_hybrid() 返回的原始结果(含 _bm25_top3
min_score: 最低分数阈值
Returns:
修改后的 contexts
"""
if not contexts:
return contexts
from config import (
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
CLUSTER_RESCUE_FLOOR,
)
if not BM25_DIVERGENCE_RESCUE_ENABLED:
return contexts
bm25_top3 = search_result.get('_bm25_top3', [])
if not bm25_top3:
return contexts
# 构建 contexts 中已有切片的 ID 索引,用于快速查找
existing_ids = {}
for i, ctx in enumerate(contexts):
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
if chunk_id:
existing_ids[chunk_id] = i
# 计算 contexts 中的多数 source用于情况 B 注入校验)
# 防止跨文档注入无关切片(如 q013 场景2.docx 的吸烟场所切片被注入到 1.docx 的查询中)
source_counter: Dict[str, int] = {}
for ctx in contexts:
src = ctx.get('meta', {}).get('source', '')
if src:
source_counter[src] = source_counter.get(src, 0) + 1
majority_source = max(source_counter, key=source_counter.get) if source_counter else None
rescued_count = 0
for bm25_item in bm25_top3:
rank = bm25_item.get('rank', 99)
if rank > BM25_DIVERGENCE_MAX_RANK:
continue
bm25_id = bm25_item.get('id')
bm25_meta = bm25_item.get('meta', {})
bm25_doc = bm25_item.get('doc', '')
# 情况 A切片在 contexts 中但 score < min_score
if bm25_id and bm25_id in existing_ids:
ctx = contexts[existing_ids[bm25_id]]
if ctx.get('score', 0) < min_score:
ctx['score'] = CLUSTER_RESCUE_FLOOR
rescued_count += 1
logger.debug(
f"BM25 分歧救援 (情况A): rank={rank}, "
f"source={bm25_meta.get('source', '')}, "
f"section={bm25_meta.get('section', '')}, "
f"原score→{CLUSTER_RESCUE_FLOOR}"
)
continue
# 情况 B切片不在 contexts 中(被 rerank 截断或已被过滤)
# 从 _bm25_top3 备份中注入,但需校验 source 一致性
if bm25_doc and bm25_meta:
bm25_source = bm25_meta.get('source', '')
# source 一致性校验:只注入与 contexts 多数 source 一致的切片
if majority_source and bm25_source and bm25_source != majority_source:
logger.debug(
f"BM25 分歧救援 (情况B-跳过): rank={rank}, "
f"source={bm25_source} != majority={majority_source}, "
f"section={bm25_meta.get('section', '')}"
)
continue
injected_ctx = {
'doc': bm25_doc,
'meta': bm25_meta,
'score': CLUSTER_RESCUE_FLOOR,
}
contexts.append(injected_ctx)
rescued_count += 1
logger.debug(
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
f"source={bm25_meta.get('source', '')}, "
f"section={bm25_meta.get('section', '')}"
)
if rescued_count > 0:
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
return contexts
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
min_score: float) -> List[Dict]:
"""
@@ -2518,6 +2624,9 @@ def rag():
# 4. 构建 promptPhase 6LLM 图片感知)
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
# BM25 分歧检测救援BM25 top-3 但 rerank 压制的切片
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
# 词法匹配救援:当切片文本精确包含查询关键词但 CrossEncoder 评分低时,
# 提升分数使其通过 min_score 过滤(适用于独立切片无法触发聚类救援的场景)
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)

207
cleanup_orphans.py Normal file
View File

@@ -0,0 +1,207 @@
"""
孤儿文件清理工具
清理 .data/images/ 和 .data/cache/vlm/ 中不再被任何 ChromaDB 切片引用的文件。
用法:
# 预览模式(不删除,只显示孤儿文件)
python cleanup_orphans.py
# 实际删除
python cleanup_orphans.py --force
# 仅清理特定知识库
python cleanup_orphans.py --collections public_kb test_kb
"""
import argparse
import hashlib
import logging
import os
import sys
from pathlib import Path
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logger = logging.getLogger(__name__)
IMAGES_DIR = Path(".data/images")
VLM_CACHE_DIR = Path(".data/cache/vlm")
def compute_file_hash(file_path: str) -> str:
"""计算文件 MD5"""
with open(file_path, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
def collect_referenced_images(manager, collections=None) -> dict:
"""
从 ChromaDB 收集所有被引用的图片路径。
Returns:
{image_filename: set of chunk_ids referencing it}
"""
referenced = {}
if collections:
kb_names = collections
else:
kb_names = [c.name if hasattr(c, 'name') else str(c)
for c in manager.list_collections()]
for kb_name in kb_names:
try:
col = manager.get_collection(kb_name)
except Exception as e:
logger.warning(f"无法获取 {kb_name}: {e}")
continue
# 查找所有带 image_path 的切片
results = col.get(include=['metadatas'])
if not results['ids']:
continue
for chunk_id, meta in zip(results['ids'], results['metadatas']):
image_path = meta.get('image_path', '')
if not image_path:
continue
# image_path 可能是: "185a7a75d246.png" 或相对路径
filename = os.path.basename(image_path)
if filename not in referenced:
referenced[filename] = set()
referenced[filename].add(f"{kb_name}/{chunk_id}")
return referenced
def find_orphan_images(referenced: dict) -> list:
"""
查找 .data/images/ 中不再被引用的图片文件。
Returns:
[(filepath, filename, size_bytes)]
"""
orphans = []
if not IMAGES_DIR.exists():
return orphans
for f in IMAGES_DIR.iterdir():
if not f.is_file():
continue
if f.name not in referenced:
orphans.append((str(f), f.name, f.stat().st_size))
return orphans
def find_orphan_vlm_caches(referenced: dict) -> list:
"""
查找 .data/cache/vlm/ 中对应的图片已不存在的缓存文件。
缓存文件以图片 MD5 命名,如果图片被删了,缓存也应该是孤儿。
Returns:
[(filepath, filename, size_bytes)]
"""
orphans = []
if not VLM_CACHE_DIR.exists():
return orphans
# 构建 referenced 中所有图片的 MD5 集合
referenced_hashes = set()
for filename in referenced:
full_path = IMAGES_DIR / filename
if full_path.exists():
try:
img_hash = compute_file_hash(str(full_path))
referenced_hashes.add(img_hash)
except Exception:
pass
for f in VLM_CACHE_DIR.iterdir():
if not f.is_file() or not f.suffix == '.txt':
continue
cache_hash = f.stem # 文件名就是 MD5
if cache_hash not in referenced_hashes:
orphans.append((str(f), f.name, f.stat().st_size))
return orphans
def delete_files(file_list: list, dry_run: bool = True) -> int:
"""删除文件列表,返回删除数量"""
deleted = 0
for filepath, filename, size in file_list:
if dry_run:
logger.info(f" [DRY-RUN] 将删除: {filename} ({size/1024:.1f} KB)")
else:
try:
os.remove(filepath)
logger.info(f" 已删除: {filename} ({size/1024:.1f} KB)")
deleted += 1
except OSError as e:
logger.warning(f" 删除失败: {filename} - {e}")
return deleted
def main():
parser = argparse.ArgumentParser(description="孤儿文件清理工具")
parser.add_argument("--force", action="store_true",
help="实际删除文件(默认仅预览)")
parser.add_argument("--collections", nargs="+",
help="仅检查指定知识库(默认检查全部)")
args = parser.parse_args()
os.chdir(os.path.dirname(os.path.abspath(__file__)))
mode = "删除" if args.force else "预览(不删除)"
print("=" * 60)
print(f"孤儿文件清理 - {mode}")
print("=" * 60)
# 1. 收集引用
print("\n[1/4] 扫描 ChromaDB 引用...")
sys.path.insert(0, '.')
from knowledge.manager import get_kb_manager
manager = get_kb_manager()
referenced = collect_referenced_images(manager, args.collections)
print(f" 被引用的图片: {len(referenced)}")
# 2. 查找孤儿图片
print("\n[2/4] 查找孤儿图片...")
orphan_images = find_orphan_images(referenced)
total_img_size = sum(s for _, _, s in orphan_images)
print(f" 孤儿图片: {len(orphan_images)} 个 ({total_img_size/1024:.1f} KB)")
# 3. 查找孤儿 VLM 缓存
print("\n[3/4] 查找孤儿 VLM 缓存...")
orphan_caches = find_orphan_vlm_caches(referenced)
total_cache_size = sum(s for _, _, s in orphan_caches)
print(f" 孤儿缓存: {len(orphan_caches)} 个 ({total_cache_size/1024:.1f} KB)")
# 4. 清理
print("\n[4/4] 清理...")
if not orphan_images and not orphan_caches:
print(" 没有需要清理的文件")
else:
if not args.force:
print(" 预览模式,以下文件将被删除:")
img_deleted = delete_files(orphan_images, dry_run=not args.force)
cache_deleted = delete_files(orphan_caches, dry_run=not args.force)
if args.force:
print(f"\n 删除了 {img_deleted} 个图片 + {cache_deleted} 个缓存")
freed = total_img_size + total_cache_size
print(f" 释放空间: {freed/1024:.1f} KB")
else:
print(f"\n{len(orphan_images) + len(orphan_caches)} 个文件待清理")
print(f" 释放空间: {(total_img_size + total_cache_size)/1024:.1f} KB")
print(" 使用 --force 参数执行实际删除")
print("=" * 60)
if __name__ == '__main__':
main()

View File

@@ -233,10 +233,10 @@ class IntentAnalyzer:
self._exact_cache_max = 500
def _get_client(self):
"""获取 LLM 客户端"""
"""获取 LLM 客户端(百炼快速模型)"""
if self._client is None:
from config import get_llm_client
self._client = get_llm_client()
from config import get_intent_client
self._client = get_intent_client()
return self._client
def _get_cache(self):
@@ -309,9 +309,7 @@ class IntentAnalyzer:
# 2. 尝试从语义缓存获取
cache = self._get_cache()
# 短追问(如"B3类呢"严重依赖上下文embedding 区分度不足易误命中
is_short_followup = history and len(query.strip()) < 20
if cache and not is_short_followup:
if cache:
# 使用 query + 历史关键信息作为缓存键
cache_key = self._build_cache_key(query, history)
cache_emb = self._get_embedding(cache_key)
@@ -327,10 +325,7 @@ class IntentAnalyzer:
else:
logger.warning("意图分析缓存: _get_embedding 返回 None跳过缓存")
else:
if is_short_followup:
logger.debug(f"跳过语义缓存(短追问依赖上下文): query='{query[:30]}'")
elif not cache:
logger.warning("意图分析缓存: _get_cache 返回 None缓存不可用")
logger.warning("意图分析缓存: _get_cache 返回 None缓存不可用")
# 构建 prompt
user_prompt = f"""## 对话历史

View File

@@ -331,6 +331,21 @@ class CollectionMixin:
except Exception as e:
logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
# 注意:此时 ChromaDB collection 已删除cleanup_image_orphans 会扫描
# 所有剩余 collection仅该 collection 引用的图片会被识别为孤儿
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self)
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存, "
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
if kb_name in self._metadata.get("collections", {}):
del self._metadata["collections"][kb_name]
self._save_metadata()
@@ -361,8 +376,40 @@ class CollectionMixin:
self._metadata = self._load_metadata()
result = []
# 以元数据为唯一真相源,不再扫描磁盘自动补充
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
# 扫描 base_path 下的所有子目录作为向量库
# 每个向量库使用独立目录base_path/my_ky, base_path/public_kb 等
actual_collections = []
try:
if os.path.exists(self.base_path):
for item in os.listdir(self.base_path):
item_path = os.path.join(self.base_path, item)
if os.path.isdir(item_path) and not item.startswith('.'):
# 检查是否包含 chroma.sqlite3有效的向量库目录
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
actual_collections.append(item)
except Exception as e:
logger.warning(f"扫描向量库目录失败: {e}")
# 如果扫描失败,回退到元数据中的集合列表
if not actual_collections:
actual_collections = list(self._metadata.get("collections", {}).keys())
for name in actual_collections:
if name not in self._metadata.get("collections", {}):
if "collections" not in self._metadata:
self._metadata["collections"] = {}
self._metadata["collections"][name] = {
"display_name": name,
"department": "",
"description": "",
"created_at": datetime.now().isoformat()
}
logger.info(f"自动补充向量库元数据: {name}")
self._save_metadata()
stale_collections = []
for name, info in self._metadata.get("collections", {}).items():
try:
collection = self.get_collection(name)
@@ -375,18 +422,17 @@ class CollectionMixin:
description=info.get("description", "")
))
except Exception as e:
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
logger.warning(
f"跳过异常向量库 '{name}': {e}"
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
)
result.append(CollectionInfo(
name=name,
display_name=info.get("display_name", name),
document_count=0,
created_at=info.get("created_at", ""),
department=info.get("department", ""),
description=info.get("description", "")
))
stale_collections.append(name)
# 清理元数据中指向已失效集合的条目
if stale_collections:
for name in stale_collections:
self._metadata.get("collections", {}).pop(name, None)
logger.info(f"清理失效向量库元数据: {name}")
self._save_metadata()
return result

View File

@@ -72,6 +72,18 @@ class DocumentMixin:
except Exception as e:
logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
logger.info(f"{kb_name} 删除文档: {filename}, 片段数: {deleted}")
return deleted

140
knowledge/image_cleanup.py Normal file
View File

@@ -0,0 +1,140 @@
"""
图片/VLM缓存孤儿文件清理模块
提供可被 document.py / collection.py 调用的清理函数,
也可被 cleanup_orphans.py 独立脚本使用。
"""
import hashlib
import logging
import os
from pathlib import Path
logger = logging.getLogger(__name__)
IMAGES_DIR = Path(".data/images")
VLM_CACHE_DIR = Path(".data/cache/vlm")
def compute_file_hash(file_path: str) -> str:
"""计算文件 MD5"""
with open(file_path, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
def collect_referenced_images(manager, collections=None) -> set:
"""
从 ChromaDB 收集所有被引用的图片文件名。
Args:
manager: KnowledgeBaseManager 实例
collections: 限定知识库列表None 表示全部
Returns:
set of image filenames (e.g., {"185a7a75d246.png", ...})
"""
referenced = set()
if collections:
kb_names = collections
else:
kb_names = [c.name if hasattr(c, 'name') else str(c)
for c in manager.list_collections()]
for kb_name in kb_names:
try:
col = manager.get_collection(kb_name)
except Exception:
continue
results = col.get(include=['metadatas'])
if not results['ids']:
continue
for meta in results['metadatas']:
image_path = meta.get('image_path', '')
if image_path:
referenced.add(os.path.basename(image_path))
return referenced
def cleanup_image_orphans(manager, collections=None, dry_run=False) -> dict:
"""
清理不再被任何 ChromaDB 切片引用的图片和 VLM 缓存文件。
Args:
manager: KnowledgeBaseManager 实例
collections: 限定知识库列表None 表示全部
dry_run: True 时只返回孤儿列表不实际删除
Returns:
{
'orphan_images': [(filepath, filename, size_bytes)],
'orphan_caches': [(filepath, filename, size_bytes)],
'deleted_images': int,
'deleted_caches': int,
'freed_bytes': int
}
"""
result = {
'orphan_images': [],
'orphan_caches': [],
'deleted_images': 0,
'deleted_caches': 0,
'freed_bytes': 0
}
# 1. 收集引用
referenced = collect_referenced_images(manager, collections)
# 2. 查找孤儿图片
if IMAGES_DIR.exists():
for f in IMAGES_DIR.iterdir():
if f.is_file() and f.name not in referenced:
result['orphan_images'].append((str(f), f.name, f.stat().st_size))
# 3. 查找孤儿 VLM 缓存(图片已删除则缓存也应是孤儿)
if VLM_CACHE_DIR.exists():
referenced_hashes = set()
for filename in referenced:
full_path = IMAGES_DIR / filename
if full_path.exists():
try:
img_hash = compute_file_hash(str(full_path))
referenced_hashes.add(img_hash)
except Exception:
pass
for f in VLM_CACHE_DIR.iterdir():
if f.is_file() and f.suffix == '.txt':
cache_hash = f.stem
if cache_hash not in referenced_hashes:
result['orphan_caches'].append((str(f), f.name, f.stat().st_size))
# 4. 删除
if not dry_run:
for filepath, filename, size in result['orphan_images']:
try:
os.remove(filepath)
result['deleted_images'] += 1
result['freed_bytes'] += size
except OSError as e:
logger.warning(f"删除孤儿图片失败: {filename} - {e}")
for filepath, filename, size in result['orphan_caches']:
try:
os.remove(filepath)
result['deleted_caches'] += 1
result['freed_bytes'] += size
except OSError as e:
logger.warning(f"删除孤儿缓存失败: {filename} - {e}")
if result['deleted_images'] or result['deleted_caches']:
logger.info(
f"清理孤儿: {result['deleted_images']} 图片 + "
f"{result['deleted_caches']} 缓存, "
f"释放 {result['freed_bytes']/1024:.1f} KB"
)
return result

View File

@@ -654,7 +654,7 @@ class KnowledgeBaseManager(
try:
from config import get_llm_client, DASHSCOPE_MODEL
client = get_llm_client()
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
return summary.strip() if summary else ""
except Exception as e:
logger.warning(f"生成表格摘要失败: {e}")
@@ -713,13 +713,31 @@ class KnowledgeBaseManager(
]
}
],
max_tokens=512
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token需留足输出空间
)
description = response.choices[0].message.content
# 推理模型兼容content 为空时从 reasoning_content 提取
if not description or not description.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip():
import re
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if cleaned:
logger.info(f"VLM content为空从reasoning_content提取描述: {image_path}")
description = cleaned
else:
description = reasoning.strip()
if not description:
logger.warning(f"VLM 返回空描述: {image_path}")
return ""
# 缓存结果
import hashlib
import re as _re
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
cache_dir = Path('.data/cache/vlm')
cache_dir.mkdir(parents=True, exist_ok=True)

View File

@@ -616,7 +616,7 @@ class FeedbackService:
prompt=prompt,
model=self.model,
temperature=0.7,
max_tokens=200
max_tokens=512
)
if not response:

View File

@@ -159,7 +159,7 @@ class SessionManager:
SELECT id, role, content, metadata, created_at
FROM messages
WHERE session_id = ?
ORDER BY created_at DESC
ORDER BY created_at DESC, id DESC
LIMIT ?
''', (session_id, limit))