feat: 意图识别优化
This commit is contained in:
parent
f323ed5f5d
commit
ac96ab14ea
File diff suppressed because one or more lines are too long
@ -45,7 +45,7 @@ if __name__ == '__main__':
|
||||
archive_messages_mysql_repository = ArchiveMessagesRepository(db_session)
|
||||
qa_milvus_repository = QARepository()
|
||||
|
||||
state: QueryQAState = QueryQAState(original_query="学习原画好就业吗", history=['2026-07-28T17:37:14 EXTERNAL 还有 3D 场景建模的试听课，你发我一下。另外，住宿要多少钱呢？一个月\n'])
|
||||
state: QueryQAState = QueryQAState(original_query="报名有没有优惠，另外 如果和同学一起报名 有没有什么优惠", history=[])
|
||||
context = QueryQAContext(meta_mysql_repository=archive_messages_mysql_repository,
|
||||
qa_milvus_repository=qa_milvus_repository
|
||||
)
|
||||
|
||||
@ -19,6 +19,15 @@ class IntentClassificationRepository:
|
||||
if milvus_client.client is None:
|
||||
milvus_client.init()
|
||||
|
||||
def _ensure_loaded(self):
|
||||
"""确保集合已加载到内存，否则 Milvus query/search 可能返回空结果"""
|
||||
self._ensure_connection()
|
||||
try:
|
||||
milvus_client.client.load_collection(self.collection_name)
|
||||
except Exception:
|
||||
# Milvus 某些版本 load_collection 重复调用会报错，直接忽略
|
||||
pass
|
||||
|
||||
def create_collection(self, dim: int = None) -> bool:
|
||||
if dim is not None:
|
||||
self.dim = dim
|
||||
@ -55,6 +64,7 @@ class IntentClassificationRepository:
|
||||
return True
|
||||
# 迁移
|
||||
def migrate_add_review_content(self) -> bool:
|
||||
self._ensure_loaded()
|
||||
if not milvus_client.has_collection(self.collection_name):
|
||||
logger.error(f"集合 {self.collection_name} 不存在，无法迁移")
|
||||
return False
|
||||
@ -165,6 +175,7 @@ class IntentClassificationRepository:
|
||||
|
||||
async def upsert_intent_data(self, data: List[Dict[str, Any]]) -> List[int]:
|
||||
try:
|
||||
self._ensure_loaded()
|
||||
created_at = int(datetime.now().timestamp())
|
||||
all_ids = []
|
||||
for item in data:
|
||||
@ -208,6 +219,7 @@ class IntentClassificationRepository:
|
||||
return []
|
||||
|
||||
async def search_by_intent_category(self, intent_category: List[str], top_k: int = 100) -> List[Dict[str, Any]]:
|
||||
self._ensure_loaded()
|
||||
if isinstance(intent_category, str):
|
||||
intent_category = [intent_category]
|
||||
categories_str = ", ".join(f'"{c}"' for c in intent_category)
|
||||
@ -221,6 +233,7 @@ class IntentClassificationRepository:
|
||||
return result
|
||||
|
||||
async def search_by_vector(self, query_vector: List[float], top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
self._ensure_loaded()
|
||||
res = milvus_client.client.search(
|
||||
collection_name=self.collection_name,
|
||||
anns_field="content_vector",
|
||||
@ -247,6 +260,7 @@ class IntentClassificationRepository:
|
||||
return results
|
||||
|
||||
async def get_all_intent_categories(self) -> List[str]:
|
||||
self._ensure_loaded()
|
||||
result = milvus_client.client.query(
|
||||
collection_name=self.collection_name,
|
||||
output_fields=["intent_category"],
|
||||
@ -260,6 +274,7 @@ class IntentClassificationRepository:
|
||||
return [c for c in all_categories]
|
||||
|
||||
async def get_content_by_intent_category(self, intent_category: str) -> List[Dict[str, str]]:
|
||||
self._ensure_loaded()
|
||||
result = milvus_client.client.query(
|
||||
collection_name=self.collection_name,
|
||||
filter=f'intent_category == "{intent_category}"',
|
||||
@ -271,10 +286,13 @@ class IntentClassificationRepository:
|
||||
"content": item.get("content", ""),
|
||||
"review_content": item.get("review_content", "")
|
||||
}
|
||||
for item in result if item.get("content")
|
||||
# content 或 review_content 任一有非空值就保留，避免全空但 review_content 有值的被误过滤
|
||||
for item in result
|
||||
if (item.get("content") or "").strip() or (item.get("review_content") or "").strip()
|
||||
]
|
||||
|
||||
async def delete_today_data(self) -> int:
|
||||
self._ensure_loaded()
|
||||
today_start = int(datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
|
||||
query_result = milvus_client.client.query(
|
||||
collection_name=self.collection_name,
|
||||
@ -294,6 +312,7 @@ class IntentClassificationRepository:
|
||||
return len(ids_to_delete)
|
||||
|
||||
def sync_content_to_review_content(self) -> int:
|
||||
self._ensure_loaded()
|
||||
result = milvus_client.client.query(
|
||||
collection_name=self.collection_name,
|
||||
output_fields=["id", "intent_category", "content", "review_content", "content_vector", "created_at"],
|
||||
@ -337,16 +356,63 @@ async def main():
|
||||
milvus_client.init()
|
||||
repo = IntentClassificationRepository()
|
||||
|
||||
# 1. 首次部署：迁移新增 review_content 字段
|
||||
# success = repo.migrate_add_review_content()
|
||||
# print(f"迁移结果: {success}")
|
||||
# ===== 诊断输出 =====
|
||||
client = milvus_client.client
|
||||
collection_name = repo.collection_name
|
||||
|
||||
# 2. 将 content 同步写入 review_content
|
||||
# count = repo.sync_content_to_review_content()
|
||||
# print(f"同步完成，共 {count} 条")
|
||||
# 1. 集合是否存在
|
||||
has = client.has_collection(collection_name)
|
||||
print(f"[1] 集合 {collection_name} 是否存在: {has}")
|
||||
if not has:
|
||||
print(" 集合不存在，跳过后续诊断")
|
||||
return
|
||||
|
||||
# 2. 集合 describe
|
||||
desc = client.describe_collection(collection_name)
|
||||
print(f"[2] 集合信息: {desc}")
|
||||
|
||||
# 3. Milvus 关键：加载集合到内存（否则 query/search 可能返回空）
|
||||
try:
|
||||
client.load_collection(collection_name)
|
||||
print("[3] load_collection 成功")
|
||||
except Exception as e:
|
||||
print(f"[3] load_collection 提示: {e}")
|
||||
|
||||
# 4. 不指定任何 filter，直接 count / query 看总条数
|
||||
stats = client.get_collection_stats(collection_name)
|
||||
print(f"[4] 集合统计信息: {stats}")
|
||||
|
||||
# 5. 无条件 query 前 20 条 intent_category
|
||||
all_sample = client.query(
|
||||
collection_name=collection_name,
|
||||
output_fields=["id", "intent_category"],
|
||||
limit=20
|
||||
)
|
||||
print(f"[5] 前20条 intent_category 样本（共{len(all_sample)}条）:")
|
||||
for it in all_sample:
|
||||
print(f" id={it.get('id')} intent_category={repr(it.get('intent_category'))}")
|
||||
|
||||
# 6. 用优惠作为条件查询
|
||||
expr = 'intent_category == "优惠"'
|
||||
print(f"[6] 执行 filter: {expr}")
|
||||
youhui = client.query(
|
||||
collection_name=collection_name,
|
||||
filter=expr,
|
||||
output_fields=["id", "intent_category", "content", "review_content"],
|
||||
limit=100
|
||||
)
|
||||
print(f" 命中 {len(youhui)} 条")
|
||||
for it in youhui:
|
||||
c = it.get("content", "")
|
||||
rc = it.get("review_content", "")
|
||||
print(f" id={it.get('id')} intent_category={repr(it.get('intent_category'))}")
|
||||
print(f" content[:100]={repr(c[:100] if c else c)} len={len(c)}")
|
||||
print(f" review_content[:100]={repr(rc[:100] if rc else rc)} len={len(rc)}")
|
||||
|
||||
# 7. 原有方法调用
|
||||
style_categories = await repo.get_content_by_intent_category("优惠")
|
||||
print(f"[7] repo.get_content_by_intent_category('优惠') 结果: {style_categories}")
|
||||
|
||||
style_categories = await repo.get_style_intent_categories()
|
||||
print(style_categories)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
Loading…
Reference in New Issue
Block a user