123 lines
4.7 KiB
Python
123 lines
4.7 KiB
Python
"""
|
||
数据清洗，把提取的问题进行聚类，然后只提取一个
|
||
"""
|
||
import json
|
||
from pathlib import Path
|
||
|
||
from langchain_core.output_parsers import StrOutputParser
|
||
from langchain_core.prompts import PromptTemplate
|
||
from sklearn.cluster import DBSCAN
|
||
import numpy as np
|
||
import asyncio
|
||
from app.client.embedding_client_manager import EmbeddingClientManager, embedding_client
|
||
from app.conf.app_config import app_config
|
||
from app.llm import llm_low
|
||
|
||
qa_file = Path(r"E:\qa_result.jsonl") # E:\qa_output\qa_result.jsonl
|
||
|
||
qa_list = []
|
||
with open(qa_file, 'r', encoding='utf-8') as f:
|
||
for line_num, line in enumerate(f, 1):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
data = json.loads(line)
|
||
if isinstance(data, list):
|
||
qa_list.extend(data)
|
||
elif isinstance(data, dict):
|
||
qa_list.append(data)
|
||
else:
|
||
print(f"警告: 第{line_num}行格式异常，跳过")
|
||
except json.JSONDecodeError as e:
|
||
print(f"警告: 第{line_num}行JSON解析失败: {e}，跳过")
|
||
|
||
print(f"成功读取 {len(qa_list)} 条QA数据")
|
||
|
||
# qa_list = [
|
||
# {"question": "教学软件是否为全流程制作，课前需要准备哪些设备？", "answer": "需要准备手绘板与手绘笔，手绘笔通常为配套。"},
|
||
# {"question": "教学软件是否为全流程制作，课前需要准备哪些设备", "answer": "需要准备手绘板与手绘笔，手绘笔通常为配套"},
|
||
# # ... 更多数据
|
||
# ]
|
||
# 2. 对question进行向量化（异步调用远程模型）
|
||
questions = [item["question"] for item in qa_list]
|
||
|
||
async def build()->list:
|
||
try:
|
||
embedding_client.init()
|
||
# embeddings_batch = await embedding_client.aembed_documents(questions)
|
||
# embeddings = [np.array(emb) for emb in embeddings_batch]
|
||
embedding_batch_size = 20
|
||
embedding_ed_all: list[list[float]] = []
|
||
for i in range(0, len(questions), embedding_batch_size):
|
||
embedding_part_texts = questions[i:i + embedding_batch_size]
|
||
embedding_eds = await embedding_client.aembed_documents(embedding_part_texts)
|
||
embedding_ed_all.extend(embedding_eds)
|
||
|
||
# 3. DBSCAN聚类（基于余弦相似度，eps≈0.15-0.25）
|
||
# 相似度 = 1 - cosine_distance，eps=0.2 约等于相似度>0.8
|
||
clustering = DBSCAN(eps=0.3, min_samples=1, metric='cosine').fit(embedding_ed_all)
|
||
|
||
# 3. 组内LLM择优
|
||
results = []
|
||
unique_labels = set(clustering.labels_)
|
||
for label in unique_labels:
|
||
group = [qa_list[i] for i, l in enumerate(clustering.labels_) if l == label]
|
||
if len(group) == 1:
|
||
results.append(group[0])
|
||
else:
|
||
best = await select_best_qa(group)
|
||
results.append(best)
|
||
print(f"组内去重：从 {len(group)} 条中保留最优，舍弃 {len(group) - 1} 条")
|
||
|
||
except Exception as e:
|
||
print(f"处理失败: {str(e)}")
|
||
finally:
|
||
await embedding_client.close()
|
||
print("-----------------------------------------")
|
||
for item in results:
|
||
print(item)
|
||
return results
|
||
|
||
|
||
async def select_best_qa(qa_group: list) -> str:
|
||
"""
|
||
从一组相似QA中，选出最完整、信息最丰富的一个
|
||
"""
|
||
# 构建对比文本
|
||
# qa_text = "\n\n".join([
|
||
# f"【选项{i + 1}】\n问题：{item['question']}\n回答：{item['answer']}"
|
||
# for i, item in enumerate(qa_group)
|
||
# ])
|
||
qa_text = "\n\n".join([
|
||
f"question: {item['question']},answer: {item['answer']}"
|
||
for i, item in enumerate(qa_group)
|
||
])
|
||
prompt = """你是一位数据清洗专家。以下是一组语义相似的QA，请从中选出**最完整、信息最丰富、对用户最有价值**的一个。
|
||
|
||
评判标准（按优先级排序）：
|
||
1. **回答完整性**：回答内容是否详尽，覆盖的信息点更多
|
||
2. **问题明确度**：问题表述是否清晰、无歧义
|
||
3. **信息密度**：单位字数内有效信息更多，无冗余废话
|
||
4. **实用性**：对用户实际解决问题的帮助程度
|
||
{qa_text}
|
||
请直接输出最完整结果。格式：{"question": "xxxxx", "answer": "bbbb"}
|
||
"""
|
||
|
||
prompt = PromptTemplate(template=prompt, input_variables=["qa_text"])
|
||
chain = prompt | llm_low | StrOutputParser()
|
||
response = await chain.ainvoke({"qa_text": qa_text})
|
||
print(response)
|
||
return response
|
||
# # 解析结果，返回对应QA
|
||
# content = response.choices[0].message.content
|
||
# # 简单解析逻辑...
|
||
# import re
|
||
# match = re.search(r'选项(\d+)', content)
|
||
# best_idx = int(match.group(1)) - 1 if match else 0
|
||
#
|
||
# return qa_group[best_idx]
|
||
|
||
if __name__ == '__main__':
|
||
asyncio.run(build())
|