异常优化处理
This commit is contained in:
parent
addbde2e26
commit
8ad68d5724
15
app/llm.py
15
app/llm.py
@ -1,5 +1,18 @@
|
|||||||
from langchain.chat_models import init_chat_model
|
from langchain.chat_models import init_chat_model
|
||||||
|
from httpx import Client, Timeout
|
||||||
|
|
||||||
from app.conf.app_config import app_config
|
from app.conf.app_config import app_config
|
||||||
|
|
||||||
llm = init_chat_model(model=app_config.llm.model_name, api_key=app_config.llm.api_key, base_url=app_config.llm.base_url ,temperature=1,extra_body={"thinking": {"type": "disabled"}},)
|
http_client = Client(
|
||||||
|
timeout=Timeout(120.0, connect=10.0, read=60.0),
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = init_chat_model(model=app_config.llm.model_name,
|
||||||
|
api_key=app_config.llm.api_key,
|
||||||
|
base_url=app_config.llm.base_url,
|
||||||
|
temperature=1,
|
||||||
|
extra_body={"thinking": {"type": "disabled"}},
|
||||||
|
max_retries=3,
|
||||||
|
request_timeout=120,
|
||||||
|
http_client=http_client)
|
||||||
|
|||||||
@ -22,24 +22,24 @@ class ArchiveMessagesRepository:
|
|||||||
ArchiveMessages 对象列表
|
ArchiveMessages 对象列表
|
||||||
"""
|
"""
|
||||||
base_query = """
|
base_query = """
|
||||||
SELECT DATE(created_at) AS created_at, from_user
|
SELECT DATE(FROM_UNIXTIME(msgtime / 1000)) AS created_at, from_user
|
||||||
FROM archive_messages
|
FROM archive_messages
|
||||||
"""
|
"""
|
||||||
conditions = []
|
conditions = []
|
||||||
params = {}
|
params = {}
|
||||||
|
|
||||||
if start_date:
|
if start_date:
|
||||||
conditions.append(" and DATE(created_at) >= :start_date")
|
conditions.append(" and DATE(FROM_UNIXTIME(msgtime / 1000)) >= :start_date")
|
||||||
params["start_date"] = start_date
|
params["start_date"] = start_date
|
||||||
|
|
||||||
if end_date:
|
if end_date:
|
||||||
conditions.append("DATE(created_at) <= :end_date")
|
conditions.append("DATE(FROM_UNIXTIME(msgtime / 1000)) <= :end_date")
|
||||||
params["end_date"] = end_date
|
params["end_date"] = end_date
|
||||||
base_query += " WHERE 1=1 and roomid='' and msgtype='text' "
|
base_query += " WHERE 1=1 and roomid='' and msgtype='text' "
|
||||||
#base_query += " WHERE (from_user='wmI1AkDQAA3h0jxrRpeaHZhMKeHExA4w' )or( from_user='LiHeYi' and to_user='[\"wmI1AkDQAA3h0jxrRpeaHZhMKeHExA4w\"]') and roomid='' and msgtype='text' "
|
#base_query += " WHERE (from_user='wmI1AkDQAA3h0jxrRpeaHZhMKeHExA4w' )or( from_user='LiHeYi' and to_user='[\"wmI1AkDQAA3h0jxrRpeaHZhMKeHExA4w\"]') and roomid='' and msgtype='text' "
|
||||||
if conditions:
|
if conditions:
|
||||||
base_query += " AND ".join(conditions)+" "
|
base_query += " AND ".join(conditions)+" "
|
||||||
base_query += " GROUP BY DATE(created_at), from_user ORDER BY created_at DESC, from_user;"
|
base_query += " GROUP BY DATE(FROM_UNIXTIME(msgtime / 1000)), from_user ORDER BY created_at DESC, from_user;"
|
||||||
|
|
||||||
query = text(base_query)
|
query = text(base_query)
|
||||||
result = await self.session.execute(query, params)
|
result = await self.session.execute(query, params)
|
||||||
@ -55,7 +55,7 @@ class ArchiveMessagesRepository:
|
|||||||
if to_user:
|
if to_user:
|
||||||
sql = """
|
sql = """
|
||||||
SELECT * FROM archive_messages
|
SELECT * FROM archive_messages
|
||||||
WHERE DATE(created_at) = :day
|
WHERE DATE(FROM_UNIXTIME(msgtime / 1000)) = :day
|
||||||
AND from_user = :from_user
|
AND from_user = :from_user
|
||||||
AND to_user = :to_user and msgtype='text'
|
AND to_user = :to_user and msgtype='text'
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
@ -64,7 +64,7 @@ class ArchiveMessagesRepository:
|
|||||||
else :
|
else :
|
||||||
sql = """
|
sql = """
|
||||||
SELECT from_user,to_user FROM archive_messages
|
SELECT from_user,to_user FROM archive_messages
|
||||||
WHERE DATE(created_at) = :day
|
WHERE DATE(FROM_UNIXTIME(msgtime / 1000)) = :day
|
||||||
AND from_user = :from_user and msgtype='text'
|
AND from_user = :from_user and msgtype='text'
|
||||||
group by to_user
|
group by to_user
|
||||||
"""
|
"""
|
||||||
|
|||||||
@ -71,6 +71,9 @@ class SummaryService:
|
|||||||
coro_list = list1 + list2
|
coro_list = list1 + list2
|
||||||
coro_list.sort(key=lambda x: x.created_at, reverse=False)
|
coro_list.sort(key=lambda x: x.created_at, reverse=False)
|
||||||
logger.info(f"查询{date_user_a}和{date_user_b}对话共:{len(coro_list)}")
|
logger.info(f"查询{date_user_a}和{date_user_b}对话共:{len(coro_list)}")
|
||||||
|
filter_gossip_message_text:str= ""
|
||||||
|
summary_text:str = ""
|
||||||
|
try:
|
||||||
# 保留语义骨架,去除闲聊填充
|
# 保留语义骨架,去除闲聊填充
|
||||||
filter_gossip_message_res = await self.filter_gossip_message(coro_list)
|
filter_gossip_message_res = await self.filter_gossip_message(coro_list)
|
||||||
filter_gossip_message_text = str(filter_gossip_message_res.content) if hasattr(
|
filter_gossip_message_text = str(filter_gossip_message_res.content) if hasattr(
|
||||||
@ -80,11 +83,12 @@ class SummaryService:
|
|||||||
summary_res = await self.date_message_summary(coro_list)
|
summary_res = await self.date_message_summary(coro_list)
|
||||||
summary_text = str(summary_res.content) if hasattr(summary_res, 'content') else str(summary_res)
|
summary_text = str(summary_res.content) if hasattr(summary_res, 'content') else str(summary_res)
|
||||||
logger.info(f"查询{date_user_a}和{date_user_b}对话进行摘要")
|
logger.info(f"查询{date_user_a}和{date_user_b}对话进行摘要")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"操作LLM失败: {e}")
|
||||||
# 摘要向量
|
# 摘要向量
|
||||||
batch_embeddings = await self.embedding_client.aembed_documents(
|
batch_embeddings = await self.embedding_client.aembed_documents(
|
||||||
[summary_text, filter_gossip_message_text])
|
[summary_text, filter_gossip_message_text])
|
||||||
|
|
||||||
#
|
|
||||||
try:
|
try:
|
||||||
self.summary_repository.insert(
|
self.summary_repository.insert(
|
||||||
msg_time=[day_date],
|
msg_time=[day_date],
|
||||||
@ -99,7 +103,7 @@ class SummaryService:
|
|||||||
)
|
)
|
||||||
logger.info(f"查询{date_user_a}和{date_user_b}对话成功入库")
|
logger.info(f"查询{date_user_a}和{date_user_b}对话成功入库")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"插入Milvus失败: {e}")
|
logger.error(f"操作milvus失败: {e}")
|
||||||
logger.info(f"运行结束:{(time.time()-stime)}")
|
logger.info(f"运行结束:{(time.time()-stime)}")
|
||||||
|
|
||||||
async def filter_gossip_message(self, coro_list: List[ArchiveMessages]) -> str:
|
async def filter_gossip_message(self, coro_list: List[ArchiveMessages]) -> str:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user