sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)
同步内容: - knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback - services/feedback.py: FAQ max_tokens 200→512 - services/session.py: 消息排序增加 id DESC 次级排序 - knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块 - knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略 - knowledge/document.py: 文档删除时清理孤儿文件 - api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援) - core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑 - cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
@@ -293,6 +293,112 @@ def _process_table_doc(doc: str, meta: Dict) -> str:
|
|||||||
return doc
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
|
||||||
|
min_score: float) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
BM25-CrossEncoder 分歧检测救援:当 BM25 排名靠前(top-3)的切片
|
||||||
|
被 CrossEncoder rerank 压制(分数低于 min_score 或被截断)时,
|
||||||
|
将其分数提升至保底值,使其通过后续的 min_score 过滤。
|
||||||
|
|
||||||
|
适用场景:
|
||||||
|
- BM25 top-1/top-2 的切片(精确关键词匹配强信号)被 rerank 评分极低
|
||||||
|
- 正确切片在 rerank 后被截断,根本不在 contexts 中
|
||||||
|
|
||||||
|
Args:
|
||||||
|
contexts: 全部上下文切片
|
||||||
|
search_result: engine.search_hybrid() 返回的原始结果(含 _bm25_top3)
|
||||||
|
min_score: 最低分数阈值
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
修改后的 contexts
|
||||||
|
"""
|
||||||
|
if not contexts:
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
from config import (
|
||||||
|
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
|
||||||
|
CLUSTER_RESCUE_FLOOR,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not BM25_DIVERGENCE_RESCUE_ENABLED:
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
bm25_top3 = search_result.get('_bm25_top3', [])
|
||||||
|
if not bm25_top3:
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
# 构建 contexts 中已有切片的 ID 索引,用于快速查找
|
||||||
|
existing_ids = {}
|
||||||
|
for i, ctx in enumerate(contexts):
|
||||||
|
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
|
||||||
|
if chunk_id:
|
||||||
|
existing_ids[chunk_id] = i
|
||||||
|
|
||||||
|
# 计算 contexts 中的多数 source(用于情况 B 注入校验)
|
||||||
|
# 防止跨文档注入无关切片(如 q013 场景:2.docx 的吸烟场所切片被注入到 1.docx 的查询中)
|
||||||
|
source_counter: Dict[str, int] = {}
|
||||||
|
for ctx in contexts:
|
||||||
|
src = ctx.get('meta', {}).get('source', '')
|
||||||
|
if src:
|
||||||
|
source_counter[src] = source_counter.get(src, 0) + 1
|
||||||
|
majority_source = max(source_counter, key=source_counter.get) if source_counter else None
|
||||||
|
|
||||||
|
rescued_count = 0
|
||||||
|
|
||||||
|
for bm25_item in bm25_top3:
|
||||||
|
rank = bm25_item.get('rank', 99)
|
||||||
|
if rank > BM25_DIVERGENCE_MAX_RANK:
|
||||||
|
continue
|
||||||
|
|
||||||
|
bm25_id = bm25_item.get('id')
|
||||||
|
bm25_meta = bm25_item.get('meta', {})
|
||||||
|
bm25_doc = bm25_item.get('doc', '')
|
||||||
|
|
||||||
|
# 情况 A:切片在 contexts 中但 score < min_score
|
||||||
|
if bm25_id and bm25_id in existing_ids:
|
||||||
|
ctx = contexts[existing_ids[bm25_id]]
|
||||||
|
if ctx.get('score', 0) < min_score:
|
||||||
|
ctx['score'] = CLUSTER_RESCUE_FLOOR
|
||||||
|
rescued_count += 1
|
||||||
|
logger.debug(
|
||||||
|
f"BM25 分歧救援 (情况A): rank={rank}, "
|
||||||
|
f"source={bm25_meta.get('source', '')}, "
|
||||||
|
f"section={bm25_meta.get('section', '')}, "
|
||||||
|
f"原score→{CLUSTER_RESCUE_FLOOR}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 情况 B:切片不在 contexts 中(被 rerank 截断或已被过滤)
|
||||||
|
# 从 _bm25_top3 备份中注入,但需校验 source 一致性
|
||||||
|
if bm25_doc and bm25_meta:
|
||||||
|
bm25_source = bm25_meta.get('source', '')
|
||||||
|
# source 一致性校验:只注入与 contexts 多数 source 一致的切片
|
||||||
|
if majority_source and bm25_source and bm25_source != majority_source:
|
||||||
|
logger.debug(
|
||||||
|
f"BM25 分歧救援 (情况B-跳过): rank={rank}, "
|
||||||
|
f"source={bm25_source} != majority={majority_source}, "
|
||||||
|
f"section={bm25_meta.get('section', '')}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
injected_ctx = {
|
||||||
|
'doc': bm25_doc,
|
||||||
|
'meta': bm25_meta,
|
||||||
|
'score': CLUSTER_RESCUE_FLOOR,
|
||||||
|
}
|
||||||
|
contexts.append(injected_ctx)
|
||||||
|
rescued_count += 1
|
||||||
|
logger.debug(
|
||||||
|
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
|
||||||
|
f"source={bm25_meta.get('source', '')}, "
|
||||||
|
f"section={bm25_meta.get('section', '')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if rescued_count > 0:
|
||||||
|
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
|
||||||
|
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
|
||||||
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
|
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
|
||||||
min_score: float) -> List[Dict]:
|
min_score: float) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
@@ -2518,6 +2624,9 @@ def rag():
|
|||||||
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
||||||
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
||||||
|
|
||||||
|
# BM25 分歧检测救援:BM25 top-3 但 rerank 压制的切片
|
||||||
|
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
|
||||||
|
|
||||||
# 词法匹配救援:当切片文本精确包含查询关键词但 CrossEncoder 评分低时,
|
# 词法匹配救援:当切片文本精确包含查询关键词但 CrossEncoder 评分低时,
|
||||||
# 提升分数使其通过 min_score 过滤(适用于独立切片无法触发聚类救援的场景)
|
# 提升分数使其通过 min_score 过滤(适用于独立切片无法触发聚类救援的场景)
|
||||||
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
|
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
|
||||||
|
|||||||
207
cleanup_orphans.py
Normal file
207
cleanup_orphans.py
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
"""
|
||||||
|
孤儿文件清理工具
|
||||||
|
|
||||||
|
清理 .data/images/ 和 .data/cache/vlm/ 中不再被任何 ChromaDB 切片引用的文件。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
# 预览模式(不删除,只显示孤儿文件)
|
||||||
|
python cleanup_orphans.py
|
||||||
|
|
||||||
|
# 实际删除
|
||||||
|
python cleanup_orphans.py --force
|
||||||
|
|
||||||
|
# 仅清理特定知识库
|
||||||
|
python cleanup_orphans.py --collections public_kb test_kb
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
IMAGES_DIR = Path(".data/images")
|
||||||
|
VLM_CACHE_DIR = Path(".data/cache/vlm")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_file_hash(file_path: str) -> str:
|
||||||
|
"""计算文件 MD5"""
|
||||||
|
with open(file_path, 'rb') as f:
|
||||||
|
return hashlib.md5(f.read()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def collect_referenced_images(manager, collections=None) -> dict:
|
||||||
|
"""
|
||||||
|
从 ChromaDB 收集所有被引用的图片路径。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{image_filename: set of chunk_ids referencing it}
|
||||||
|
"""
|
||||||
|
referenced = {}
|
||||||
|
|
||||||
|
if collections:
|
||||||
|
kb_names = collections
|
||||||
|
else:
|
||||||
|
kb_names = [c.name if hasattr(c, 'name') else str(c)
|
||||||
|
for c in manager.list_collections()]
|
||||||
|
|
||||||
|
for kb_name in kb_names:
|
||||||
|
try:
|
||||||
|
col = manager.get_collection(kb_name)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"无法获取 {kb_name}: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 查找所有带 image_path 的切片
|
||||||
|
results = col.get(include=['metadatas'])
|
||||||
|
if not results['ids']:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for chunk_id, meta in zip(results['ids'], results['metadatas']):
|
||||||
|
image_path = meta.get('image_path', '')
|
||||||
|
if not image_path:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# image_path 可能是: "185a7a75d246.png" 或相对路径
|
||||||
|
filename = os.path.basename(image_path)
|
||||||
|
if filename not in referenced:
|
||||||
|
referenced[filename] = set()
|
||||||
|
referenced[filename].add(f"{kb_name}/{chunk_id}")
|
||||||
|
|
||||||
|
return referenced
|
||||||
|
|
||||||
|
|
||||||
|
def find_orphan_images(referenced: dict) -> list:
|
||||||
|
"""
|
||||||
|
查找 .data/images/ 中不再被引用的图片文件。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
[(filepath, filename, size_bytes)]
|
||||||
|
"""
|
||||||
|
orphans = []
|
||||||
|
if not IMAGES_DIR.exists():
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
for f in IMAGES_DIR.iterdir():
|
||||||
|
if not f.is_file():
|
||||||
|
continue
|
||||||
|
if f.name not in referenced:
|
||||||
|
orphans.append((str(f), f.name, f.stat().st_size))
|
||||||
|
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
|
||||||
|
def find_orphan_vlm_caches(referenced: dict) -> list:
|
||||||
|
"""
|
||||||
|
查找 .data/cache/vlm/ 中对应的图片已不存在的缓存文件。
|
||||||
|
|
||||||
|
缓存文件以图片 MD5 命名,如果图片被删了,缓存也应该是孤儿。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
[(filepath, filename, size_bytes)]
|
||||||
|
"""
|
||||||
|
orphans = []
|
||||||
|
if not VLM_CACHE_DIR.exists():
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
# 构建 referenced 中所有图片的 MD5 集合
|
||||||
|
referenced_hashes = set()
|
||||||
|
for filename in referenced:
|
||||||
|
full_path = IMAGES_DIR / filename
|
||||||
|
if full_path.exists():
|
||||||
|
try:
|
||||||
|
img_hash = compute_file_hash(str(full_path))
|
||||||
|
referenced_hashes.add(img_hash)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
for f in VLM_CACHE_DIR.iterdir():
|
||||||
|
if not f.is_file() or not f.suffix == '.txt':
|
||||||
|
continue
|
||||||
|
cache_hash = f.stem # 文件名就是 MD5
|
||||||
|
if cache_hash not in referenced_hashes:
|
||||||
|
orphans.append((str(f), f.name, f.stat().st_size))
|
||||||
|
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
|
||||||
|
def delete_files(file_list: list, dry_run: bool = True) -> int:
|
||||||
|
"""删除文件列表,返回删除数量"""
|
||||||
|
deleted = 0
|
||||||
|
for filepath, filename, size in file_list:
|
||||||
|
if dry_run:
|
||||||
|
logger.info(f" [DRY-RUN] 将删除: {filename} ({size/1024:.1f} KB)")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
logger.info(f" 已删除: {filename} ({size/1024:.1f} KB)")
|
||||||
|
deleted += 1
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f" 删除失败: {filename} - {e}")
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="孤儿文件清理工具")
|
||||||
|
parser.add_argument("--force", action="store_true",
|
||||||
|
help="实际删除文件(默认仅预览)")
|
||||||
|
parser.add_argument("--collections", nargs="+",
|
||||||
|
help="仅检查指定知识库(默认检查全部)")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
mode = "删除" if args.force else "预览(不删除)"
|
||||||
|
print("=" * 60)
|
||||||
|
print(f"孤儿文件清理 - {mode}")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
# 1. 收集引用
|
||||||
|
print("\n[1/4] 扫描 ChromaDB 引用...")
|
||||||
|
sys.path.insert(0, '.')
|
||||||
|
from knowledge.manager import get_kb_manager
|
||||||
|
manager = get_kb_manager()
|
||||||
|
referenced = collect_referenced_images(manager, args.collections)
|
||||||
|
print(f" 被引用的图片: {len(referenced)} 个")
|
||||||
|
|
||||||
|
# 2. 查找孤儿图片
|
||||||
|
print("\n[2/4] 查找孤儿图片...")
|
||||||
|
orphan_images = find_orphan_images(referenced)
|
||||||
|
total_img_size = sum(s for _, _, s in orphan_images)
|
||||||
|
print(f" 孤儿图片: {len(orphan_images)} 个 ({total_img_size/1024:.1f} KB)")
|
||||||
|
|
||||||
|
# 3. 查找孤儿 VLM 缓存
|
||||||
|
print("\n[3/4] 查找孤儿 VLM 缓存...")
|
||||||
|
orphan_caches = find_orphan_vlm_caches(referenced)
|
||||||
|
total_cache_size = sum(s for _, _, s in orphan_caches)
|
||||||
|
print(f" 孤儿缓存: {len(orphan_caches)} 个 ({total_cache_size/1024:.1f} KB)")
|
||||||
|
|
||||||
|
# 4. 清理
|
||||||
|
print("\n[4/4] 清理...")
|
||||||
|
if not orphan_images and not orphan_caches:
|
||||||
|
print(" 没有需要清理的文件")
|
||||||
|
else:
|
||||||
|
if not args.force:
|
||||||
|
print(" 预览模式,以下文件将被删除:")
|
||||||
|
|
||||||
|
img_deleted = delete_files(orphan_images, dry_run=not args.force)
|
||||||
|
cache_deleted = delete_files(orphan_caches, dry_run=not args.force)
|
||||||
|
|
||||||
|
if args.force:
|
||||||
|
print(f"\n 删除了 {img_deleted} 个图片 + {cache_deleted} 个缓存")
|
||||||
|
freed = total_img_size + total_cache_size
|
||||||
|
print(f" 释放空间: {freed/1024:.1f} KB")
|
||||||
|
else:
|
||||||
|
print(f"\n 共 {len(orphan_images) + len(orphan_caches)} 个文件待清理")
|
||||||
|
print(f" 释放空间: {(total_img_size + total_cache_size)/1024:.1f} KB")
|
||||||
|
print(" 使用 --force 参数执行实际删除")
|
||||||
|
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -233,10 +233,10 @@ class IntentAnalyzer:
|
|||||||
self._exact_cache_max = 500
|
self._exact_cache_max = 500
|
||||||
|
|
||||||
def _get_client(self):
|
def _get_client(self):
|
||||||
"""获取 LLM 客户端"""
|
"""获取 LLM 客户端(百炼快速模型)"""
|
||||||
if self._client is None:
|
if self._client is None:
|
||||||
from config import get_llm_client
|
from config import get_intent_client
|
||||||
self._client = get_llm_client()
|
self._client = get_intent_client()
|
||||||
return self._client
|
return self._client
|
||||||
|
|
||||||
def _get_cache(self):
|
def _get_cache(self):
|
||||||
@@ -309,9 +309,7 @@ class IntentAnalyzer:
|
|||||||
|
|
||||||
# 2. 尝试从语义缓存获取
|
# 2. 尝试从语义缓存获取
|
||||||
cache = self._get_cache()
|
cache = self._get_cache()
|
||||||
# 短追问(如"B3类呢?")严重依赖上下文,embedding 区分度不足易误命中
|
if cache:
|
||||||
is_short_followup = history and len(query.strip()) < 20
|
|
||||||
if cache and not is_short_followup:
|
|
||||||
# 使用 query + 历史关键信息作为缓存键
|
# 使用 query + 历史关键信息作为缓存键
|
||||||
cache_key = self._build_cache_key(query, history)
|
cache_key = self._build_cache_key(query, history)
|
||||||
cache_emb = self._get_embedding(cache_key)
|
cache_emb = self._get_embedding(cache_key)
|
||||||
@@ -327,10 +325,7 @@ class IntentAnalyzer:
|
|||||||
else:
|
else:
|
||||||
logger.warning("意图分析缓存: _get_embedding 返回 None,跳过缓存")
|
logger.warning("意图分析缓存: _get_embedding 返回 None,跳过缓存")
|
||||||
else:
|
else:
|
||||||
if is_short_followup:
|
logger.warning("意图分析缓存: _get_cache 返回 None,缓存不可用")
|
||||||
logger.debug(f"跳过语义缓存(短追问依赖上下文): query='{query[:30]}'")
|
|
||||||
elif not cache:
|
|
||||||
logger.warning("意图分析缓存: _get_cache 返回 None,缓存不可用")
|
|
||||||
|
|
||||||
# 构建 prompt
|
# 构建 prompt
|
||||||
user_prompt = f"""## 对话历史
|
user_prompt = f"""## 对话历史
|
||||||
|
|||||||
@@ -331,6 +331,21 @@ class CollectionMixin:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"清理版本记录失败: {e}")
|
logger.warning(f"清理版本记录失败: {e}")
|
||||||
|
|
||||||
|
# 清理不再被引用的图片和 VLM 缓存文件
|
||||||
|
# 注意:此时 ChromaDB collection 已删除,cleanup_image_orphans 会扫描
|
||||||
|
# 所有剩余 collection,仅该 collection 引用的图片会被识别为孤儿
|
||||||
|
try:
|
||||||
|
from knowledge.image_cleanup import cleanup_image_orphans
|
||||||
|
cleanup_result = cleanup_image_orphans(self)
|
||||||
|
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||||
|
logger.info(
|
||||||
|
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||||
|
f"{cleanup_result['deleted_caches']} VLM缓存, "
|
||||||
|
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"清理孤儿文件失败: {e}")
|
||||||
|
|
||||||
if kb_name in self._metadata.get("collections", {}):
|
if kb_name in self._metadata.get("collections", {}):
|
||||||
del self._metadata["collections"][kb_name]
|
del self._metadata["collections"][kb_name]
|
||||||
self._save_metadata()
|
self._save_metadata()
|
||||||
@@ -361,8 +376,40 @@ class CollectionMixin:
|
|||||||
self._metadata = self._load_metadata()
|
self._metadata = self._load_metadata()
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
# 扫描 base_path 下的所有子目录作为向量库
|
||||||
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
||||||
|
actual_collections = []
|
||||||
|
try:
|
||||||
|
if os.path.exists(self.base_path):
|
||||||
|
for item in os.listdir(self.base_path):
|
||||||
|
item_path = os.path.join(self.base_path, item)
|
||||||
|
if os.path.isdir(item_path) and not item.startswith('.'):
|
||||||
|
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
||||||
|
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
||||||
|
actual_collections.append(item)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"扫描向量库目录失败: {e}")
|
||||||
|
|
||||||
|
# 如果扫描失败,回退到元数据中的集合列表
|
||||||
|
if not actual_collections:
|
||||||
|
actual_collections = list(self._metadata.get("collections", {}).keys())
|
||||||
|
|
||||||
|
for name in actual_collections:
|
||||||
|
if name not in self._metadata.get("collections", {}):
|
||||||
|
if "collections" not in self._metadata:
|
||||||
|
self._metadata["collections"] = {}
|
||||||
|
self._metadata["collections"][name] = {
|
||||||
|
"display_name": name,
|
||||||
|
"department": "",
|
||||||
|
"description": "",
|
||||||
|
"created_at": datetime.now().isoformat()
|
||||||
|
}
|
||||||
|
logger.info(f"自动补充向量库元数据: {name}")
|
||||||
|
|
||||||
|
self._save_metadata()
|
||||||
|
|
||||||
|
stale_collections = []
|
||||||
|
|
||||||
for name, info in self._metadata.get("collections", {}).items():
|
for name, info in self._metadata.get("collections", {}).items():
|
||||||
try:
|
try:
|
||||||
collection = self.get_collection(name)
|
collection = self.get_collection(name)
|
||||||
@@ -375,18 +422,17 @@ class CollectionMixin:
|
|||||||
description=info.get("description", "")
|
description=info.get("description", "")
|
||||||
))
|
))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"跳过异常向量库 '{name}': {e}"
|
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
|
||||||
)
|
)
|
||||||
result.append(CollectionInfo(
|
stale_collections.append(name)
|
||||||
name=name,
|
|
||||||
display_name=info.get("display_name", name),
|
# 清理元数据中指向已失效集合的条目
|
||||||
document_count=0,
|
if stale_collections:
|
||||||
created_at=info.get("created_at", ""),
|
for name in stale_collections:
|
||||||
department=info.get("department", ""),
|
self._metadata.get("collections", {}).pop(name, None)
|
||||||
description=info.get("description", "")
|
logger.info(f"清理失效向量库元数据: {name}")
|
||||||
))
|
self._save_metadata()
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -72,6 +72,18 @@ class DocumentMixin:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"清理版本记录失败: {e}")
|
logger.warning(f"清理版本记录失败: {e}")
|
||||||
|
|
||||||
|
# 清理不再被引用的图片和 VLM 缓存文件
|
||||||
|
try:
|
||||||
|
from knowledge.image_cleanup import cleanup_image_orphans
|
||||||
|
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
|
||||||
|
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||||
|
logger.info(
|
||||||
|
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||||
|
f"{cleanup_result['deleted_caches']} VLM缓存"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"清理孤儿文件失败: {e}")
|
||||||
|
|
||||||
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
||||||
return deleted
|
return deleted
|
||||||
|
|
||||||
|
|||||||
140
knowledge/image_cleanup.py
Normal file
140
knowledge/image_cleanup.py
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
"""
|
||||||
|
图片/VLM缓存孤儿文件清理模块
|
||||||
|
|
||||||
|
提供可被 document.py / collection.py 调用的清理函数,
|
||||||
|
也可被 cleanup_orphans.py 独立脚本使用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
IMAGES_DIR = Path(".data/images")
|
||||||
|
VLM_CACHE_DIR = Path(".data/cache/vlm")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_file_hash(file_path: str) -> str:
|
||||||
|
"""计算文件 MD5"""
|
||||||
|
with open(file_path, 'rb') as f:
|
||||||
|
return hashlib.md5(f.read()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def collect_referenced_images(manager, collections=None) -> set:
|
||||||
|
"""
|
||||||
|
从 ChromaDB 收集所有被引用的图片文件名。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
manager: KnowledgeBaseManager 实例
|
||||||
|
collections: 限定知识库列表,None 表示全部
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
set of image filenames (e.g., {"185a7a75d246.png", ...})
|
||||||
|
"""
|
||||||
|
referenced = set()
|
||||||
|
|
||||||
|
if collections:
|
||||||
|
kb_names = collections
|
||||||
|
else:
|
||||||
|
kb_names = [c.name if hasattr(c, 'name') else str(c)
|
||||||
|
for c in manager.list_collections()]
|
||||||
|
|
||||||
|
for kb_name in kb_names:
|
||||||
|
try:
|
||||||
|
col = manager.get_collection(kb_name)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
results = col.get(include=['metadatas'])
|
||||||
|
if not results['ids']:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for meta in results['metadatas']:
|
||||||
|
image_path = meta.get('image_path', '')
|
||||||
|
if image_path:
|
||||||
|
referenced.add(os.path.basename(image_path))
|
||||||
|
|
||||||
|
return referenced
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_image_orphans(manager, collections=None, dry_run=False) -> dict:
|
||||||
|
"""
|
||||||
|
清理不再被任何 ChromaDB 切片引用的图片和 VLM 缓存文件。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
manager: KnowledgeBaseManager 实例
|
||||||
|
collections: 限定知识库列表,None 表示全部
|
||||||
|
dry_run: True 时只返回孤儿列表不实际删除
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
'orphan_images': [(filepath, filename, size_bytes)],
|
||||||
|
'orphan_caches': [(filepath, filename, size_bytes)],
|
||||||
|
'deleted_images': int,
|
||||||
|
'deleted_caches': int,
|
||||||
|
'freed_bytes': int
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
result = {
|
||||||
|
'orphan_images': [],
|
||||||
|
'orphan_caches': [],
|
||||||
|
'deleted_images': 0,
|
||||||
|
'deleted_caches': 0,
|
||||||
|
'freed_bytes': 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# 1. 收集引用
|
||||||
|
referenced = collect_referenced_images(manager, collections)
|
||||||
|
|
||||||
|
# 2. 查找孤儿图片
|
||||||
|
if IMAGES_DIR.exists():
|
||||||
|
for f in IMAGES_DIR.iterdir():
|
||||||
|
if f.is_file() and f.name not in referenced:
|
||||||
|
result['orphan_images'].append((str(f), f.name, f.stat().st_size))
|
||||||
|
|
||||||
|
# 3. 查找孤儿 VLM 缓存(图片已删除则缓存也应是孤儿)
|
||||||
|
if VLM_CACHE_DIR.exists():
|
||||||
|
referenced_hashes = set()
|
||||||
|
for filename in referenced:
|
||||||
|
full_path = IMAGES_DIR / filename
|
||||||
|
if full_path.exists():
|
||||||
|
try:
|
||||||
|
img_hash = compute_file_hash(str(full_path))
|
||||||
|
referenced_hashes.add(img_hash)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
for f in VLM_CACHE_DIR.iterdir():
|
||||||
|
if f.is_file() and f.suffix == '.txt':
|
||||||
|
cache_hash = f.stem
|
||||||
|
if cache_hash not in referenced_hashes:
|
||||||
|
result['orphan_caches'].append((str(f), f.name, f.stat().st_size))
|
||||||
|
|
||||||
|
# 4. 删除
|
||||||
|
if not dry_run:
|
||||||
|
for filepath, filename, size in result['orphan_images']:
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
result['deleted_images'] += 1
|
||||||
|
result['freed_bytes'] += size
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"删除孤儿图片失败: {filename} - {e}")
|
||||||
|
|
||||||
|
for filepath, filename, size in result['orphan_caches']:
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
result['deleted_caches'] += 1
|
||||||
|
result['freed_bytes'] += size
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"删除孤儿缓存失败: {filename} - {e}")
|
||||||
|
|
||||||
|
if result['deleted_images'] or result['deleted_caches']:
|
||||||
|
logger.info(
|
||||||
|
f"清理孤儿: {result['deleted_images']} 图片 + "
|
||||||
|
f"{result['deleted_caches']} 缓存, "
|
||||||
|
f"释放 {result['freed_bytes']/1024:.1f} KB"
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
@@ -654,7 +654,7 @@ class KnowledgeBaseManager(
|
|||||||
try:
|
try:
|
||||||
from config import get_llm_client, DASHSCOPE_MODEL
|
from config import get_llm_client, DASHSCOPE_MODEL
|
||||||
client = get_llm_client()
|
client = get_llm_client()
|
||||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
|
||||||
return summary.strip() if summary else ""
|
return summary.strip() if summary else ""
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"生成表格摘要失败: {e}")
|
logger.warning(f"生成表格摘要失败: {e}")
|
||||||
@@ -713,13 +713,31 @@ class KnowledgeBaseManager(
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
max_tokens=512
|
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间
|
||||||
)
|
)
|
||||||
|
|
||||||
description = response.choices[0].message.content
|
description = response.choices[0].message.content
|
||||||
|
|
||||||
|
# 推理模型兼容:content 为空时从 reasoning_content 提取
|
||||||
|
if not description or not description.strip():
|
||||||
|
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||||
|
if reasoning and reasoning.strip():
|
||||||
|
import re
|
||||||
|
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
|
||||||
|
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||||
|
if cleaned:
|
||||||
|
logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}")
|
||||||
|
description = cleaned
|
||||||
|
else:
|
||||||
|
description = reasoning.strip()
|
||||||
|
|
||||||
|
if not description:
|
||||||
|
logger.warning(f"VLM 返回空描述: {image_path}")
|
||||||
|
return ""
|
||||||
|
|
||||||
# 缓存结果
|
# 缓存结果
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import re as _re
|
||||||
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
||||||
cache_dir = Path('.data/cache/vlm')
|
cache_dir = Path('.data/cache/vlm')
|
||||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|||||||
@@ -616,7 +616,7 @@ class FeedbackService:
|
|||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
temperature=0.7,
|
temperature=0.7,
|
||||||
max_tokens=200
|
max_tokens=512
|
||||||
)
|
)
|
||||||
|
|
||||||
if not response:
|
if not response:
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ class SessionManager:
|
|||||||
SELECT id, role, content, metadata, created_at
|
SELECT id, role, content, metadata, created_at
|
||||||
FROM messages
|
FROM messages
|
||||||
WHERE session_id = ?
|
WHERE session_id = ?
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC, id DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
''', (session_id, limit))
|
''', (session_id, limit))
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user