Compare commits
4 Commits
server-bas
...
148559ee3c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
148559ee3c | ||
|
|
431af0217a | ||
|
|
aa04bb94a6 | ||
|
|
84a8be0ce8 |
23
.dockerignore
Normal file
23
.dockerignore
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# 大体积数据目录(通过 volume 挂载,不需要打进镜像)
|
||||||
|
models/
|
||||||
|
knowledge/vector_store/
|
||||||
|
documents/
|
||||||
|
.data/
|
||||||
|
data/
|
||||||
|
|
||||||
|
# Python 虚拟环境
|
||||||
|
venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Git
|
||||||
|
.git/
|
||||||
|
|
||||||
|
# IDE 和编辑器
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
|
||||||
|
# 其他
|
||||||
|
*.log
|
||||||
|
.env*
|
||||||
@@ -134,6 +134,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
from .base import BM25Index
|
from .base import BM25Index
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if not kb_name or not kb_name.replace('_', '').isalnum():
|
if not kb_name or not kb_name.replace('_', '').isalnum():
|
||||||
return False, "向量库名称只能包含字母、数字和下划线"
|
return False, "向量库名称只能包含字母、数字和下划线"
|
||||||
|
|
||||||
@@ -190,6 +193,9 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
更新成功返回 True,向量库不存在返回 False
|
更新成功返回 True,向量库不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
collections = self._metadata.get("collections", {})
|
collections = self._metadata.get("collections", {})
|
||||||
if kb_name not in collections:
|
if kb_name not in collections:
|
||||||
return False
|
return False
|
||||||
@@ -228,6 +234,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if kb_name == PUBLIC_KB_NAME:
|
if kb_name == PUBLIC_KB_NAME:
|
||||||
return False, "公开知识库不能删除"
|
return False, "公开知识库不能删除"
|
||||||
|
|
||||||
@@ -348,41 +357,14 @@ class CollectionMixin:
|
|||||||
- department: 所属部门
|
- department: 所属部门
|
||||||
- description: 描述
|
- description: 描述
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
# 扫描 base_path 下的所有子目录作为向量库
|
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
||||||
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
||||||
actual_collections = []
|
|
||||||
try:
|
|
||||||
if os.path.exists(self.base_path):
|
|
||||||
for item in os.listdir(self.base_path):
|
|
||||||
item_path = os.path.join(self.base_path, item)
|
|
||||||
if os.path.isdir(item_path) and not item.startswith('.'):
|
|
||||||
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
|
||||||
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
|
||||||
actual_collections.append(item)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"扫描向量库目录失败: {e}")
|
|
||||||
|
|
||||||
# 如果扫描失败,回退到元数据中的集合列表
|
|
||||||
if not actual_collections:
|
|
||||||
actual_collections = list(self._metadata.get("collections", {}).keys())
|
|
||||||
|
|
||||||
for name in actual_collections:
|
|
||||||
if name not in self._metadata.get("collections", {}):
|
|
||||||
if "collections" not in self._metadata:
|
|
||||||
self._metadata["collections"] = {}
|
|
||||||
self._metadata["collections"][name] = {
|
|
||||||
"display_name": name,
|
|
||||||
"department": "",
|
|
||||||
"description": "",
|
|
||||||
"created_at": datetime.now().isoformat()
|
|
||||||
}
|
|
||||||
logger.info(f"自动补充向量库元数据: {name}")
|
|
||||||
|
|
||||||
self._save_metadata()
|
|
||||||
|
|
||||||
for name, info in self._metadata.get("collections", {}).items():
|
for name, info in self._metadata.get("collections", {}).items():
|
||||||
|
try:
|
||||||
collection = self.get_collection(name)
|
collection = self.get_collection(name)
|
||||||
result.append(CollectionInfo(
|
result.append(CollectionInfo(
|
||||||
name=name,
|
name=name,
|
||||||
@@ -392,6 +374,19 @@ class CollectionMixin:
|
|||||||
department=info.get("department", ""),
|
department=info.get("department", ""),
|
||||||
description=info.get("description", "")
|
description=info.get("description", "")
|
||||||
))
|
))
|
||||||
|
except Exception as e:
|
||||||
|
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
||||||
|
logger.warning(
|
||||||
|
f"跳过异常向量库 '{name}': {e}"
|
||||||
|
)
|
||||||
|
result.append(CollectionInfo(
|
||||||
|
name=name,
|
||||||
|
display_name=info.get("display_name", name),
|
||||||
|
document_count=0,
|
||||||
|
created_at=info.get("created_at", ""),
|
||||||
|
department=info.get("department", ""),
|
||||||
|
description=info.get("description", "")
|
||||||
|
))
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -405,4 +400,6 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
存在返回 True,不存在返回 False
|
存在返回 True,不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
return kb_name in self._metadata.get("collections", {})
|
return kb_name in self._metadata.get("collections", {})
|
||||||
|
|||||||
@@ -29,6 +29,11 @@
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import threading
|
import threading
|
||||||
|
try:
|
||||||
|
import fcntl
|
||||||
|
_HAS_FCNTL = True
|
||||||
|
except ImportError:
|
||||||
|
_HAS_FCNTL = False
|
||||||
from typing import List, Dict, Optional, Tuple
|
from typing import List, Dict, Optional, Tuple
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import logging
|
import logging
|
||||||
@@ -134,22 +139,36 @@ class KnowledgeBaseManager(
|
|||||||
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
||||||
|
|
||||||
def _load_metadata(self) -> dict:
|
def _load_metadata(self) -> dict:
|
||||||
"""加载元数据"""
|
"""加载元数据(带文件锁)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
if os.path.exists(metadata_path):
|
if os.path.exists(metadata_path):
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
|
||||||
|
try:
|
||||||
return json.load(f)
|
return json.load(f)
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"加载元数据失败: {e}")
|
logger.error(f"加载元数据失败: {e}")
|
||||||
return {"collections": {}}
|
return {"collections": {}}
|
||||||
|
|
||||||
def _save_metadata(self):
|
def _save_metadata(self):
|
||||||
"""保存元数据"""
|
"""保存元数据(带文件锁,防止并发写入覆盖)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'w', encoding='utf-8') as f:
|
with open(metadata_path, 'w', encoding='utf-8') as f:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
||||||
|
f.flush()
|
||||||
|
os.fsync(f.fileno())
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"保存元数据失败: {e}")
|
logger.error(f"保存元数据失败: {e}")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user