mirror of
https://github.com/xszyou/Fay.git
synced 2026-03-12 17:51:28 +08:00
配置模块
1、修复配置加载逻辑。
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
import re
|
||||
import logging
|
||||
import re
|
||||
import requests
|
||||
from typing import List, Optional
|
||||
import threading
|
||||
@@ -20,19 +20,19 @@ except ImportError as e:
|
||||
|
||||
# 使用统一日志配置
|
||||
from bionicmemory.utils.logging_config import get_logger
|
||||
logger = get_logger(__name__)
|
||||
|
||||
if not CONFIG_UTIL_AVAILABLE:
|
||||
logger.warning("无法导入 config_util,将使用环境变量配置")
|
||||
|
||||
def _sanitize_text(text: str) -> str:
|
||||
if not isinstance(text, str) or not text:
|
||||
return text
|
||||
cleaned = re.sub(r'<think>[\s\S]*?</think>', '', text, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r'</?think>', '', cleaned, flags=re.IGNORECASE)
|
||||
return cleaned
|
||||
|
||||
class ApiEmbeddingService:
|
||||
logger = get_logger(__name__)
|
||||
|
||||
if not CONFIG_UTIL_AVAILABLE:
|
||||
logger.warning("无法导入 config_util,将使用环境变量配置")
|
||||
|
||||
def _sanitize_text(text: str) -> str:
|
||||
if not isinstance(text, str) or not text:
|
||||
return text
|
||||
cleaned = re.sub(r'<think>[\s\S]*?</think>', '', text, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r'</?think>', '', cleaned, flags=re.IGNORECASE)
|
||||
return cleaned
|
||||
|
||||
class ApiEmbeddingService:
|
||||
"""API Embedding服务 - 单例模式,调用 OpenAI 兼容的 API"""
|
||||
|
||||
_instance = None
|
||||
@@ -120,14 +120,14 @@ class ApiEmbeddingService:
|
||||
logger.error(f"API Embedding 服务初始化失败: {e}")
|
||||
raise
|
||||
|
||||
def encode_text(self, text: str) -> List[float]:
|
||||
"""编码单个文本(带重试机制)"""
|
||||
import time
|
||||
|
||||
text = _sanitize_text(text)
|
||||
last_error = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
def encode_text(self, text: str) -> List[float]:
|
||||
"""编码单个文本(带重试机制)"""
|
||||
import time
|
||||
|
||||
text = _sanitize_text(text)
|
||||
last_error = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
# 调用 API 进行编码
|
||||
url = f"{self.api_base_url}/embeddings"
|
||||
headers = {
|
||||
@@ -152,7 +152,7 @@ class ApiEmbeddingService:
|
||||
# 首次调用时获取实际维度
|
||||
if self.embedding_dim is None:
|
||||
self.embedding_dim = len(embedding)
|
||||
logger.info(f"动态获取 embedding 维度: {self.embedding_dim}")
|
||||
logger.info(f"动态获取 embedding 维度: {self.embedding_dim},与原记忆节点不一致,将重新生成记忆节点的 embedding维度")
|
||||
|
||||
logger.info(f"embedding 生成成功")
|
||||
return embedding
|
||||
@@ -178,12 +178,12 @@ class ApiEmbeddingService:
|
||||
else:
|
||||
raise
|
||||
|
||||
def encode_texts(self, texts: List[str]) -> List[List[float]]:
|
||||
"""批量编码文本"""
|
||||
try:
|
||||
texts = [_sanitize_text(text) for text in texts]
|
||||
# 调用 API 进行批量编码
|
||||
url = f"{self.api_base_url}/embeddings"
|
||||
def encode_texts(self, texts: List[str]) -> List[List[float]]:
|
||||
"""批量编码文本"""
|
||||
try:
|
||||
texts = [_sanitize_text(text) for text in texts]
|
||||
# 调用 API 进行批量编码
|
||||
url = f"{self.api_base_url}/embeddings"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}"
|
||||
|
||||
@@ -64,6 +64,7 @@ embedding_api_key = None
|
||||
_last_loaded_project_id = None
|
||||
_last_loaded_config = None
|
||||
_last_loaded_from_api = False # 表示上次加载来自配置中心(含缓存)
|
||||
_bootstrap_loaded_from_api = False # 无本地配置时启动阶段已从配置中心加载过
|
||||
_warned_public_project_ids = set()
|
||||
|
||||
# config server中心配置,system.conf与config.json存在时不会使用配置中心
|
||||
@@ -205,11 +206,13 @@ def load_config(force_reload=False):
|
||||
global _last_loaded_project_id
|
||||
global _last_loaded_config
|
||||
global _last_loaded_from_api
|
||||
global _bootstrap_loaded_from_api
|
||||
|
||||
_refresh_config_center()
|
||||
|
||||
env_project_id = os.getenv('FAY_CONFIG_CENTER_ID')
|
||||
using_config_center = bool(env_project_id)
|
||||
explicit_config_center = bool(env_project_id)
|
||||
using_config_center = explicit_config_center
|
||||
if (
|
||||
env_project_id
|
||||
and not force_reload
|
||||
@@ -223,8 +226,12 @@ def load_config(force_reload=False):
|
||||
default_config_json_path = os.path.join(os.getcwd(), 'config.json')
|
||||
cache_system_conf_path = os.path.join(os.getcwd(), 'cache_data', 'system.conf')
|
||||
cache_config_json_path = os.path.join(os.getcwd(), 'cache_data', 'config.json')
|
||||
root_system_conf_exists = os.path.exists(default_system_conf_path)
|
||||
root_config_json_exists = os.path.exists(default_config_json_path)
|
||||
root_config_complete = root_system_conf_exists and root_config_json_exists
|
||||
|
||||
# 构建system.conf和config.json的完整路径
|
||||
config_center_fallback = False
|
||||
if using_config_center:
|
||||
system_conf_path = cache_system_conf_path
|
||||
config_json_path = cache_config_json_path
|
||||
@@ -238,11 +245,25 @@ def load_config(force_reload=False):
|
||||
system_conf_path = default_system_conf_path
|
||||
config_json_path = default_config_json_path
|
||||
|
||||
if not root_config_complete:
|
||||
cache_ready = os.path.exists(cache_system_conf_path) and os.path.exists(cache_config_json_path)
|
||||
if (not _bootstrap_loaded_from_api) or (not cache_ready):
|
||||
using_config_center = True
|
||||
config_center_fallback = True
|
||||
system_conf_path = cache_system_conf_path
|
||||
config_json_path = cache_config_json_path
|
||||
else:
|
||||
system_conf_path = cache_system_conf_path
|
||||
config_json_path = cache_config_json_path
|
||||
|
||||
forced_loaded = False
|
||||
loaded_from_api = False
|
||||
api_attempted = False
|
||||
if using_config_center:
|
||||
util.log(1, f"检测到配置中心参数,优先加载项目配置: {CONFIG_SERVER['PROJECT_ID']}")
|
||||
if explicit_config_center:
|
||||
util.log(1, f"检测到配置中心参数,优先加载项目配置: {CONFIG_SERVER['PROJECT_ID']}")
|
||||
else:
|
||||
util.log(1, f"未检测到本地system.conf或config.json,尝试从配置中心加载配置: {CONFIG_SERVER['PROJECT_ID']}")
|
||||
api_config = load_config_from_api(CONFIG_SERVER['PROJECT_ID'])
|
||||
api_attempted = True
|
||||
if api_config:
|
||||
@@ -250,6 +271,8 @@ def load_config(force_reload=False):
|
||||
system_config = api_config['system_config']
|
||||
config = api_config['config']
|
||||
loaded_from_api = True
|
||||
if config_center_fallback:
|
||||
_bootstrap_loaded_from_api = True
|
||||
|
||||
# 缓存API配置到本地文件
|
||||
system_conf_path = cache_system_conf_path
|
||||
@@ -281,6 +304,8 @@ def load_config(force_reload=False):
|
||||
system_config = api_config['system_config']
|
||||
config = api_config['config']
|
||||
loaded_from_api = True
|
||||
if config_center_fallback:
|
||||
_bootstrap_loaded_from_api = True
|
||||
|
||||
# 缓存API配置到本地文件
|
||||
system_conf_path = cache_system_conf_path
|
||||
@@ -322,7 +347,21 @@ def load_config(force_reload=False):
|
||||
if _last_loaded_config is not None and _last_loaded_from_api:
|
||||
util.log(2, "配置中心缓存不可用,继续使用内存中的配置")
|
||||
return _last_loaded_config
|
||||
# 如果本地文件存在,从本地文件加载
|
||||
if config_center_fallback and using_config_center and (not sys_conf_exists or not config_json_exists):
|
||||
cache_ready = os.path.exists(cache_system_conf_path) and os.path.exists(cache_config_json_path)
|
||||
if cache_ready:
|
||||
util.log(2, "配置中心不可用,回退使用缓存配置")
|
||||
using_config_center = False
|
||||
system_conf_path = cache_system_conf_path
|
||||
config_json_path = cache_config_json_path
|
||||
else:
|
||||
util.log(2, "配置中心不可用,回退使用本地配置文件")
|
||||
using_config_center = False
|
||||
system_conf_path = default_system_conf_path
|
||||
config_json_path = default_config_json_path
|
||||
sys_conf_exists = os.path.exists(system_conf_path)
|
||||
config_json_exists = os.path.exists(config_json_path)
|
||||
# 如果本地文件存在,从本地文件加载
|
||||
# 加载system.conf
|
||||
system_config = ConfigParser()
|
||||
system_config.read(system_conf_path, encoding='UTF-8')
|
||||
|
||||
Reference in New Issue
Block a user