sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)
同步内容: - knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback - services/feedback.py: FAQ max_tokens 200→512 - services/session.py: 消息排序增加 id DESC 次级排序 - knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块 - knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略 - knowledge/document.py: 文档删除时清理孤儿文件 - api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援) - core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑 - cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
@@ -293,6 +293,112 @@ def _process_table_doc(doc: str, meta: Dict) -> str:
|
||||
return doc
|
||||
|
||||
|
||||
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
BM25-CrossEncoder 分歧检测救援:当 BM25 排名靠前(top-3)的切片
|
||||
被 CrossEncoder rerank 压制(分数低于 min_score 或被截断)时,
|
||||
将其分数提升至保底值,使其通过后续的 min_score 过滤。
|
||||
|
||||
适用场景:
|
||||
- BM25 top-1/top-2 的切片(精确关键词匹配强信号)被 rerank 评分极低
|
||||
- 正确切片在 rerank 后被截断,根本不在 contexts 中
|
||||
|
||||
Args:
|
||||
contexts: 全部上下文切片
|
||||
search_result: engine.search_hybrid() 返回的原始结果(含 _bm25_top3)
|
||||
min_score: 最低分数阈值
|
||||
|
||||
Returns:
|
||||
修改后的 contexts
|
||||
"""
|
||||
if not contexts:
|
||||
return contexts
|
||||
|
||||
from config import (
|
||||
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
|
||||
CLUSTER_RESCUE_FLOOR,
|
||||
)
|
||||
|
||||
if not BM25_DIVERGENCE_RESCUE_ENABLED:
|
||||
return contexts
|
||||
|
||||
bm25_top3 = search_result.get('_bm25_top3', [])
|
||||
if not bm25_top3:
|
||||
return contexts
|
||||
|
||||
# 构建 contexts 中已有切片的 ID 索引,用于快速查找
|
||||
existing_ids = {}
|
||||
for i, ctx in enumerate(contexts):
|
||||
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
|
||||
if chunk_id:
|
||||
existing_ids[chunk_id] = i
|
||||
|
||||
# 计算 contexts 中的多数 source(用于情况 B 注入校验)
|
||||
# 防止跨文档注入无关切片(如 q013 场景:2.docx 的吸烟场所切片被注入到 1.docx 的查询中)
|
||||
source_counter: Dict[str, int] = {}
|
||||
for ctx in contexts:
|
||||
src = ctx.get('meta', {}).get('source', '')
|
||||
if src:
|
||||
source_counter[src] = source_counter.get(src, 0) + 1
|
||||
majority_source = max(source_counter, key=source_counter.get) if source_counter else None
|
||||
|
||||
rescued_count = 0
|
||||
|
||||
for bm25_item in bm25_top3:
|
||||
rank = bm25_item.get('rank', 99)
|
||||
if rank > BM25_DIVERGENCE_MAX_RANK:
|
||||
continue
|
||||
|
||||
bm25_id = bm25_item.get('id')
|
||||
bm25_meta = bm25_item.get('meta', {})
|
||||
bm25_doc = bm25_item.get('doc', '')
|
||||
|
||||
# 情况 A:切片在 contexts 中但 score < min_score
|
||||
if bm25_id and bm25_id in existing_ids:
|
||||
ctx = contexts[existing_ids[bm25_id]]
|
||||
if ctx.get('score', 0) < min_score:
|
||||
ctx['score'] = CLUSTER_RESCUE_FLOOR
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况A): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}, "
|
||||
f"原score→{CLUSTER_RESCUE_FLOOR}"
|
||||
)
|
||||
continue
|
||||
|
||||
# 情况 B:切片不在 contexts 中(被 rerank 截断或已被过滤)
|
||||
# 从 _bm25_top3 备份中注入,但需校验 source 一致性
|
||||
if bm25_doc and bm25_meta:
|
||||
bm25_source = bm25_meta.get('source', '')
|
||||
# source 一致性校验:只注入与 contexts 多数 source 一致的切片
|
||||
if majority_source and bm25_source and bm25_source != majority_source:
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况B-跳过): rank={rank}, "
|
||||
f"source={bm25_source} != majority={majority_source}, "
|
||||
f"section={bm25_meta.get('section', '')}"
|
||||
)
|
||||
continue
|
||||
injected_ctx = {
|
||||
'doc': bm25_doc,
|
||||
'meta': bm25_meta,
|
||||
'score': CLUSTER_RESCUE_FLOOR,
|
||||
}
|
||||
contexts.append(injected_ctx)
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}"
|
||||
)
|
||||
|
||||
if rescued_count > 0:
|
||||
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
|
||||
|
||||
return contexts
|
||||
|
||||
|
||||
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
@@ -2518,6 +2624,9 @@ def rag():
|
||||
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
||||
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
||||
|
||||
# BM25 分歧检测救援:BM25 top-3 但 rerank 压制的切片
|
||||
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
# 词法匹配救援:当切片文本精确包含查询关键词但 CrossEncoder 评分低时,
|
||||
# 提升分数使其通过 min_score 过滤(适用于独立切片无法触发聚类救援的场景)
|
||||
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
207
cleanup_orphans.py
Normal file
207
cleanup_orphans.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
孤儿文件清理工具
|
||||
|
||||
清理 .data/images/ 和 .data/cache/vlm/ 中不再被任何 ChromaDB 切片引用的文件。
|
||||
|
||||
用法:
|
||||
# 预览模式(不删除,只显示孤儿文件)
|
||||
python cleanup_orphans.py
|
||||
|
||||
# 实际删除
|
||||
python cleanup_orphans.py --force
|
||||
|
||||
# 仅清理特定知识库
|
||||
python cleanup_orphans.py --collections public_kb test_kb
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGES_DIR = Path(".data/images")
|
||||
VLM_CACHE_DIR = Path(".data/cache/vlm")
|
||||
|
||||
|
||||
def compute_file_hash(file_path: str) -> str:
|
||||
"""计算文件 MD5"""
|
||||
with open(file_path, 'rb') as f:
|
||||
return hashlib.md5(f.read()).hexdigest()
|
||||
|
||||
|
||||
def collect_referenced_images(manager, collections=None) -> dict:
|
||||
"""
|
||||
从 ChromaDB 收集所有被引用的图片路径。
|
||||
|
||||
Returns:
|
||||
{image_filename: set of chunk_ids referencing it}
|
||||
"""
|
||||
referenced = {}
|
||||
|
||||
if collections:
|
||||
kb_names = collections
|
||||
else:
|
||||
kb_names = [c.name if hasattr(c, 'name') else str(c)
|
||||
for c in manager.list_collections()]
|
||||
|
||||
for kb_name in kb_names:
|
||||
try:
|
||||
col = manager.get_collection(kb_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"无法获取 {kb_name}: {e}")
|
||||
continue
|
||||
|
||||
# 查找所有带 image_path 的切片
|
||||
results = col.get(include=['metadatas'])
|
||||
if not results['ids']:
|
||||
continue
|
||||
|
||||
for chunk_id, meta in zip(results['ids'], results['metadatas']):
|
||||
image_path = meta.get('image_path', '')
|
||||
if not image_path:
|
||||
continue
|
||||
|
||||
# image_path 可能是: "185a7a75d246.png" 或相对路径
|
||||
filename = os.path.basename(image_path)
|
||||
if filename not in referenced:
|
||||
referenced[filename] = set()
|
||||
referenced[filename].add(f"{kb_name}/{chunk_id}")
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def find_orphan_images(referenced: dict) -> list:
|
||||
"""
|
||||
查找 .data/images/ 中不再被引用的图片文件。
|
||||
|
||||
Returns:
|
||||
[(filepath, filename, size_bytes)]
|
||||
"""
|
||||
orphans = []
|
||||
if not IMAGES_DIR.exists():
|
||||
return orphans
|
||||
|
||||
for f in IMAGES_DIR.iterdir():
|
||||
if not f.is_file():
|
||||
continue
|
||||
if f.name not in referenced:
|
||||
orphans.append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
return orphans
|
||||
|
||||
|
||||
def find_orphan_vlm_caches(referenced: dict) -> list:
|
||||
"""
|
||||
查找 .data/cache/vlm/ 中对应的图片已不存在的缓存文件。
|
||||
|
||||
缓存文件以图片 MD5 命名,如果图片被删了,缓存也应该是孤儿。
|
||||
|
||||
Returns:
|
||||
[(filepath, filename, size_bytes)]
|
||||
"""
|
||||
orphans = []
|
||||
if not VLM_CACHE_DIR.exists():
|
||||
return orphans
|
||||
|
||||
# 构建 referenced 中所有图片的 MD5 集合
|
||||
referenced_hashes = set()
|
||||
for filename in referenced:
|
||||
full_path = IMAGES_DIR / filename
|
||||
if full_path.exists():
|
||||
try:
|
||||
img_hash = compute_file_hash(str(full_path))
|
||||
referenced_hashes.add(img_hash)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for f in VLM_CACHE_DIR.iterdir():
|
||||
if not f.is_file() or not f.suffix == '.txt':
|
||||
continue
|
||||
cache_hash = f.stem # 文件名就是 MD5
|
||||
if cache_hash not in referenced_hashes:
|
||||
orphans.append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
return orphans
|
||||
|
||||
|
||||
def delete_files(file_list: list, dry_run: bool = True) -> int:
|
||||
"""删除文件列表,返回删除数量"""
|
||||
deleted = 0
|
||||
for filepath, filename, size in file_list:
|
||||
if dry_run:
|
||||
logger.info(f" [DRY-RUN] 将删除: {filename} ({size/1024:.1f} KB)")
|
||||
else:
|
||||
try:
|
||||
os.remove(filepath)
|
||||
logger.info(f" 已删除: {filename} ({size/1024:.1f} KB)")
|
||||
deleted += 1
|
||||
except OSError as e:
|
||||
logger.warning(f" 删除失败: {filename} - {e}")
|
||||
return deleted
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="孤儿文件清理工具")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="实际删除文件(默认仅预览)")
|
||||
parser.add_argument("--collections", nargs="+",
|
||||
help="仅检查指定知识库(默认检查全部)")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
mode = "删除" if args.force else "预览(不删除)"
|
||||
print("=" * 60)
|
||||
print(f"孤儿文件清理 - {mode}")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. 收集引用
|
||||
print("\n[1/4] 扫描 ChromaDB 引用...")
|
||||
sys.path.insert(0, '.')
|
||||
from knowledge.manager import get_kb_manager
|
||||
manager = get_kb_manager()
|
||||
referenced = collect_referenced_images(manager, args.collections)
|
||||
print(f" 被引用的图片: {len(referenced)} 个")
|
||||
|
||||
# 2. 查找孤儿图片
|
||||
print("\n[2/4] 查找孤儿图片...")
|
||||
orphan_images = find_orphan_images(referenced)
|
||||
total_img_size = sum(s for _, _, s in orphan_images)
|
||||
print(f" 孤儿图片: {len(orphan_images)} 个 ({total_img_size/1024:.1f} KB)")
|
||||
|
||||
# 3. 查找孤儿 VLM 缓存
|
||||
print("\n[3/4] 查找孤儿 VLM 缓存...")
|
||||
orphan_caches = find_orphan_vlm_caches(referenced)
|
||||
total_cache_size = sum(s for _, _, s in orphan_caches)
|
||||
print(f" 孤儿缓存: {len(orphan_caches)} 个 ({total_cache_size/1024:.1f} KB)")
|
||||
|
||||
# 4. 清理
|
||||
print("\n[4/4] 清理...")
|
||||
if not orphan_images and not orphan_caches:
|
||||
print(" 没有需要清理的文件")
|
||||
else:
|
||||
if not args.force:
|
||||
print(" 预览模式,以下文件将被删除:")
|
||||
|
||||
img_deleted = delete_files(orphan_images, dry_run=not args.force)
|
||||
cache_deleted = delete_files(orphan_caches, dry_run=not args.force)
|
||||
|
||||
if args.force:
|
||||
print(f"\n 删除了 {img_deleted} 个图片 + {cache_deleted} 个缓存")
|
||||
freed = total_img_size + total_cache_size
|
||||
print(f" 释放空间: {freed/1024:.1f} KB")
|
||||
else:
|
||||
print(f"\n 共 {len(orphan_images) + len(orphan_caches)} 个文件待清理")
|
||||
print(f" 释放空间: {(total_img_size + total_cache_size)/1024:.1f} KB")
|
||||
print(" 使用 --force 参数执行实际删除")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -233,10 +233,10 @@ class IntentAnalyzer:
|
||||
self._exact_cache_max = 500
|
||||
|
||||
def _get_client(self):
|
||||
"""获取 LLM 客户端"""
|
||||
"""获取 LLM 客户端(百炼快速模型)"""
|
||||
if self._client is None:
|
||||
from config import get_llm_client
|
||||
self._client = get_llm_client()
|
||||
from config import get_intent_client
|
||||
self._client = get_intent_client()
|
||||
return self._client
|
||||
|
||||
def _get_cache(self):
|
||||
@@ -309,9 +309,7 @@ class IntentAnalyzer:
|
||||
|
||||
# 2. 尝试从语义缓存获取
|
||||
cache = self._get_cache()
|
||||
# 短追问(如"B3类呢?")严重依赖上下文,embedding 区分度不足易误命中
|
||||
is_short_followup = history and len(query.strip()) < 20
|
||||
if cache and not is_short_followup:
|
||||
if cache:
|
||||
# 使用 query + 历史关键信息作为缓存键
|
||||
cache_key = self._build_cache_key(query, history)
|
||||
cache_emb = self._get_embedding(cache_key)
|
||||
@@ -327,10 +325,7 @@ class IntentAnalyzer:
|
||||
else:
|
||||
logger.warning("意图分析缓存: _get_embedding 返回 None,跳过缓存")
|
||||
else:
|
||||
if is_short_followup:
|
||||
logger.debug(f"跳过语义缓存(短追问依赖上下文): query='{query[:30]}'")
|
||||
elif not cache:
|
||||
logger.warning("意图分析缓存: _get_cache 返回 None,缓存不可用")
|
||||
logger.warning("意图分析缓存: _get_cache 返回 None,缓存不可用")
|
||||
|
||||
# 构建 prompt
|
||||
user_prompt = f"""## 对话历史
|
||||
|
||||
@@ -331,6 +331,21 @@ class CollectionMixin:
|
||||
except Exception as e:
|
||||
logger.warning(f"清理版本记录失败: {e}")
|
||||
|
||||
# 清理不再被引用的图片和 VLM 缓存文件
|
||||
# 注意:此时 ChromaDB collection 已删除,cleanup_image_orphans 会扫描
|
||||
# 所有剩余 collection,仅该 collection 引用的图片会被识别为孤儿
|
||||
try:
|
||||
from knowledge.image_cleanup import cleanup_image_orphans
|
||||
cleanup_result = cleanup_image_orphans(self)
|
||||
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||
logger.info(
|
||||
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||
f"{cleanup_result['deleted_caches']} VLM缓存, "
|
||||
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理孤儿文件失败: {e}")
|
||||
|
||||
if kb_name in self._metadata.get("collections", {}):
|
||||
del self._metadata["collections"][kb_name]
|
||||
self._save_metadata()
|
||||
@@ -361,8 +376,40 @@ class CollectionMixin:
|
||||
self._metadata = self._load_metadata()
|
||||
result = []
|
||||
|
||||
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
||||
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
||||
# 扫描 base_path 下的所有子目录作为向量库
|
||||
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
||||
actual_collections = []
|
||||
try:
|
||||
if os.path.exists(self.base_path):
|
||||
for item in os.listdir(self.base_path):
|
||||
item_path = os.path.join(self.base_path, item)
|
||||
if os.path.isdir(item_path) and not item.startswith('.'):
|
||||
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
||||
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
||||
actual_collections.append(item)
|
||||
except Exception as e:
|
||||
logger.warning(f"扫描向量库目录失败: {e}")
|
||||
|
||||
# 如果扫描失败,回退到元数据中的集合列表
|
||||
if not actual_collections:
|
||||
actual_collections = list(self._metadata.get("collections", {}).keys())
|
||||
|
||||
for name in actual_collections:
|
||||
if name not in self._metadata.get("collections", {}):
|
||||
if "collections" not in self._metadata:
|
||||
self._metadata["collections"] = {}
|
||||
self._metadata["collections"][name] = {
|
||||
"display_name": name,
|
||||
"department": "",
|
||||
"description": "",
|
||||
"created_at": datetime.now().isoformat()
|
||||
}
|
||||
logger.info(f"自动补充向量库元数据: {name}")
|
||||
|
||||
self._save_metadata()
|
||||
|
||||
stale_collections = []
|
||||
|
||||
for name, info in self._metadata.get("collections", {}).items():
|
||||
try:
|
||||
collection = self.get_collection(name)
|
||||
@@ -375,18 +422,17 @@ class CollectionMixin:
|
||||
description=info.get("description", "")
|
||||
))
|
||||
except Exception as e:
|
||||
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
||||
logger.warning(
|
||||
f"跳过异常向量库 '{name}': {e}"
|
||||
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
|
||||
)
|
||||
result.append(CollectionInfo(
|
||||
name=name,
|
||||
display_name=info.get("display_name", name),
|
||||
document_count=0,
|
||||
created_at=info.get("created_at", ""),
|
||||
department=info.get("department", ""),
|
||||
description=info.get("description", "")
|
||||
))
|
||||
stale_collections.append(name)
|
||||
|
||||
# 清理元数据中指向已失效集合的条目
|
||||
if stale_collections:
|
||||
for name in stale_collections:
|
||||
self._metadata.get("collections", {}).pop(name, None)
|
||||
logger.info(f"清理失效向量库元数据: {name}")
|
||||
self._save_metadata()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -72,6 +72,18 @@ class DocumentMixin:
|
||||
except Exception as e:
|
||||
logger.warning(f"清理版本记录失败: {e}")
|
||||
|
||||
# 清理不再被引用的图片和 VLM 缓存文件
|
||||
try:
|
||||
from knowledge.image_cleanup import cleanup_image_orphans
|
||||
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
|
||||
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||
logger.info(
|
||||
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||
f"{cleanup_result['deleted_caches']} VLM缓存"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理孤儿文件失败: {e}")
|
||||
|
||||
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
||||
return deleted
|
||||
|
||||
|
||||
140
knowledge/image_cleanup.py
Normal file
140
knowledge/image_cleanup.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
图片/VLM缓存孤儿文件清理模块
|
||||
|
||||
提供可被 document.py / collection.py 调用的清理函数,
|
||||
也可被 cleanup_orphans.py 独立脚本使用。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGES_DIR = Path(".data/images")
|
||||
VLM_CACHE_DIR = Path(".data/cache/vlm")
|
||||
|
||||
|
||||
def compute_file_hash(file_path: str) -> str:
|
||||
"""计算文件 MD5"""
|
||||
with open(file_path, 'rb') as f:
|
||||
return hashlib.md5(f.read()).hexdigest()
|
||||
|
||||
|
||||
def collect_referenced_images(manager, collections=None) -> set:
|
||||
"""
|
||||
从 ChromaDB 收集所有被引用的图片文件名。
|
||||
|
||||
Args:
|
||||
manager: KnowledgeBaseManager 实例
|
||||
collections: 限定知识库列表,None 表示全部
|
||||
|
||||
Returns:
|
||||
set of image filenames (e.g., {"185a7a75d246.png", ...})
|
||||
"""
|
||||
referenced = set()
|
||||
|
||||
if collections:
|
||||
kb_names = collections
|
||||
else:
|
||||
kb_names = [c.name if hasattr(c, 'name') else str(c)
|
||||
for c in manager.list_collections()]
|
||||
|
||||
for kb_name in kb_names:
|
||||
try:
|
||||
col = manager.get_collection(kb_name)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
results = col.get(include=['metadatas'])
|
||||
if not results['ids']:
|
||||
continue
|
||||
|
||||
for meta in results['metadatas']:
|
||||
image_path = meta.get('image_path', '')
|
||||
if image_path:
|
||||
referenced.add(os.path.basename(image_path))
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def cleanup_image_orphans(manager, collections=None, dry_run=False) -> dict:
|
||||
"""
|
||||
清理不再被任何 ChromaDB 切片引用的图片和 VLM 缓存文件。
|
||||
|
||||
Args:
|
||||
manager: KnowledgeBaseManager 实例
|
||||
collections: 限定知识库列表,None 表示全部
|
||||
dry_run: True 时只返回孤儿列表不实际删除
|
||||
|
||||
Returns:
|
||||
{
|
||||
'orphan_images': [(filepath, filename, size_bytes)],
|
||||
'orphan_caches': [(filepath, filename, size_bytes)],
|
||||
'deleted_images': int,
|
||||
'deleted_caches': int,
|
||||
'freed_bytes': int
|
||||
}
|
||||
"""
|
||||
result = {
|
||||
'orphan_images': [],
|
||||
'orphan_caches': [],
|
||||
'deleted_images': 0,
|
||||
'deleted_caches': 0,
|
||||
'freed_bytes': 0
|
||||
}
|
||||
|
||||
# 1. 收集引用
|
||||
referenced = collect_referenced_images(manager, collections)
|
||||
|
||||
# 2. 查找孤儿图片
|
||||
if IMAGES_DIR.exists():
|
||||
for f in IMAGES_DIR.iterdir():
|
||||
if f.is_file() and f.name not in referenced:
|
||||
result['orphan_images'].append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
# 3. 查找孤儿 VLM 缓存(图片已删除则缓存也应是孤儿)
|
||||
if VLM_CACHE_DIR.exists():
|
||||
referenced_hashes = set()
|
||||
for filename in referenced:
|
||||
full_path = IMAGES_DIR / filename
|
||||
if full_path.exists():
|
||||
try:
|
||||
img_hash = compute_file_hash(str(full_path))
|
||||
referenced_hashes.add(img_hash)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for f in VLM_CACHE_DIR.iterdir():
|
||||
if f.is_file() and f.suffix == '.txt':
|
||||
cache_hash = f.stem
|
||||
if cache_hash not in referenced_hashes:
|
||||
result['orphan_caches'].append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
# 4. 删除
|
||||
if not dry_run:
|
||||
for filepath, filename, size in result['orphan_images']:
|
||||
try:
|
||||
os.remove(filepath)
|
||||
result['deleted_images'] += 1
|
||||
result['freed_bytes'] += size
|
||||
except OSError as e:
|
||||
logger.warning(f"删除孤儿图片失败: {filename} - {e}")
|
||||
|
||||
for filepath, filename, size in result['orphan_caches']:
|
||||
try:
|
||||
os.remove(filepath)
|
||||
result['deleted_caches'] += 1
|
||||
result['freed_bytes'] += size
|
||||
except OSError as e:
|
||||
logger.warning(f"删除孤儿缓存失败: {filename} - {e}")
|
||||
|
||||
if result['deleted_images'] or result['deleted_caches']:
|
||||
logger.info(
|
||||
f"清理孤儿: {result['deleted_images']} 图片 + "
|
||||
f"{result['deleted_caches']} 缓存, "
|
||||
f"释放 {result['freed_bytes']/1024:.1f} KB"
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -654,7 +654,7 @@ class KnowledgeBaseManager(
|
||||
try:
|
||||
from config import get_llm_client, DASHSCOPE_MODEL
|
||||
client = get_llm_client()
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
|
||||
return summary.strip() if summary else ""
|
||||
except Exception as e:
|
||||
logger.warning(f"生成表格摘要失败: {e}")
|
||||
@@ -713,13 +713,31 @@ class KnowledgeBaseManager(
|
||||
]
|
||||
}
|
||||
],
|
||||
max_tokens=512
|
||||
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间
|
||||
)
|
||||
|
||||
description = response.choices[0].message.content
|
||||
|
||||
# 推理模型兼容:content 为空时从 reasoning_content 提取
|
||||
if not description or not description.strip():
|
||||
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||
if reasoning and reasoning.strip():
|
||||
import re
|
||||
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
|
||||
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||
if cleaned:
|
||||
logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}")
|
||||
description = cleaned
|
||||
else:
|
||||
description = reasoning.strip()
|
||||
|
||||
if not description:
|
||||
logger.warning(f"VLM 返回空描述: {image_path}")
|
||||
return ""
|
||||
|
||||
# 缓存结果
|
||||
import hashlib
|
||||
import re as _re
|
||||
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
||||
cache_dir = Path('.data/cache/vlm')
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -616,7 +616,7 @@ class FeedbackService:
|
||||
prompt=prompt,
|
||||
model=self.model,
|
||||
temperature=0.7,
|
||||
max_tokens=200
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
if not response:
|
||||
|
||||
@@ -159,7 +159,7 @@ class SessionManager:
|
||||
SELECT id, role, content, metadata, created_at
|
||||
FROM messages
|
||||
WHERE session_id = ?
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT ?
|
||||
''', (session_id, limit))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user