"""
数据清洗，把提取的问题进行聚类，然后只提取一个
"""
import json
from pathlib import Path

from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import PromptTemplate
from sklearn.cluster import DBSCAN
import numpy as np
import asyncio
from app.client.embedding_client_manager import EmbeddingClientManager, embedding_client
from app.conf.app_config import app_config
from app.llm import llm_low

qa_file = Path(r"E:\qa_result.jsonl")  # E:\qa_output\qa_result.jsonl

qa_list = []
with open(qa_file, 'r', encoding='utf-8') as f:
    for line_num, line in enumerate(f, 1):
        line = line.strip()
        if not line:
            continue
        try:
            data = json.loads(line)
            if isinstance(data, list):
                qa_list.extend(data)
            elif isinstance(data, dict):
                qa_list.append(data)
            else:
                print(f"警告: 第{line_num}行格式异常，跳过")
        except json.JSONDecodeError as e:
            print(f"警告: 第{line_num}行JSON解析失败: {e}，跳过")

print(f"成功读取 {len(qa_list)} 条QA数据")

# qa_list = [
#     {"question": "教学软件是否为全流程制作，课前需要准备哪些设备？", "answer": "需要准备手绘板与手绘笔，手绘笔通常为配套。"},
#     {"question": "教学软件是否为全流程制作，课前需要准备哪些设备", "answer": "需要准备手绘板与手绘笔，手绘笔通常为配套"},
#     # ... 更多数据
# ]
# 2. 对question进行向量化（异步调用远程模型）
questions = [item["question"] for item in qa_list]

async def build()->list:
    try:
        embedding_client.init()
        # embeddings_batch = await embedding_client.aembed_documents(questions)
        # embeddings = [np.array(emb) for emb in embeddings_batch]
        embedding_batch_size = 20
        embedding_ed_all: list[list[float]] = []
        for i in range(0, len(questions), embedding_batch_size):
            embedding_part_texts = questions[i:i + embedding_batch_size]
            embedding_eds = await embedding_client.aembed_documents(embedding_part_texts)
            embedding_ed_all.extend(embedding_eds)

        # 3. DBSCAN聚类（基于余弦相似度，eps≈0.15-0.25）
        # 相似度 = 1 - cosine_distance，eps=0.2 约等于相似度>0.8
        clustering = DBSCAN(eps=0.3, min_samples=1, metric='cosine').fit(embedding_ed_all)

        # 3. 组内LLM择优
        results = []
        unique_labels = set(clustering.labels_)
        for label in unique_labels:
            group = [qa_list[i] for i, l in enumerate(clustering.labels_) if l == label]
            if len(group) == 1:
                results.append(group[0])
            else:
                best = await select_best_qa(group)
                results.append(best)
                print(f"组内去重：从 {len(group)} 条中保留最优，舍弃 {len(group) - 1} 条")

    except Exception as e:
        print(f"处理失败: {str(e)}")
    finally:
        await embedding_client.close()
    print("-----------------------------------------")
    for item in results:
        print(item)
    return results


async def select_best_qa(qa_group: list) -> str:
    """
    从一组相似QA中，选出最完整、信息最丰富的一个
    """
    # 构建对比文本
    # qa_text = "\n\n".join([
    #     f"【选项{i + 1}】\n问题：{item['question']}\n回答：{item['answer']}"
    #     for i, item in enumerate(qa_group)
    # ])
    qa_text = "\n\n".join([
        f"question: {item['question']},answer: {item['answer']}"
        for i, item in enumerate(qa_group)
    ])
    prompt = """你是一位数据清洗专家。以下是一组语义相似的QA，请从中选出**最完整、信息最丰富、对用户最有价值**的一个。

评判标准（按优先级排序）：
1. **回答完整性**：回答内容是否详尽，覆盖的信息点更多
2. **问题明确度**：问题表述是否清晰、无歧义
3. **信息密度**：单位字数内有效信息更多，无冗余废话
4. **实用性**：对用户实际解决问题的帮助程度
{qa_text}
请直接输出最完整结果。格式：{"question": "xxxxx", "answer": "bbbb"}
"""

    prompt = PromptTemplate(template=prompt, input_variables=["qa_text"])
    chain = prompt | llm_low | StrOutputParser()
    response = await chain.ainvoke({"qa_text": qa_text})
    print(response)
    return response
    # # 解析结果，返回对应QA
    # content = response.choices[0].message.content
    # # 简单解析逻辑...
    # import re
    # match = re.search(r'选项(\d+)', content)
    # best_idx = int(match.group(1)) - 1 if match else 0
    #
    # return qa_group[best_idx]

if __name__ == '__main__':
    asyncio.run(build())