评估聊天机器人

export LANGSMITH_TRACING="true"
export LANGSMITH_API_KEY="<您的 LangSmith API 密钥>"
export OPENAI_API_KEY="<您的 OpenAI API 密钥>"

from langsmith import Client

client = Client()

# 定义数据集：这些是您的测试用例
dataset_name = "问答示例数据集"
dataset = client.create_dataset(dataset_name)

client.create_examples(
    dataset_id=dataset.id,
    examples=[
        {
            "inputs": {"question": "什么是 LangChain？"},
            "outputs": {"answer": "一个用于构建 LLM 应用程序的框架"},
        },
        {
            "inputs": {"question": "什么是 LangSmith？"},
            "outputs": {"answer": "一个用于观察和评估 LLM 应用程序的平台"},
        },
        {
            "inputs": {"question": "什么是 OpenAI？"},
            "outputs": {"answer": "一家创建大语言模型的公司"},
        },
        {
            "inputs": {"question": "什么是 Google？"},
            "outputs": {"answer": "一家以搜索闻名的科技公司"},
        },
        {
            "inputs": {"question": "什么是 Mistral？"},
            "outputs": {"answer": "一家创建大语言模型的公司"},
        }
    ]
)

import openai
from langsmith import wrappers

openai_client = wrappers.wrap_openai(openai.OpenAI())

eval_instructions = "您是一位专门为学生的答案评分的专家教授。"

def correctness(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
    user_content = f"""您正在为以下问题评分：
{inputs['question']}
这是真实答案：
{reference_outputs['answer']}
您正在为以下预测答案评分：
{outputs['response']}
请回答 CORRECT 或 INCORRECT：
评分："""
    response = openai_client.chat.completions.create(
        model="gpt-4.1-mini",
        temperature=0,
        messages=[
            {"role": "system", "content": eval_instructions},
            {"role": "user", "content": user_content},
        ],
    ).choices[0].message.content
    return response == "CORRECT"

def concision(outputs: dict, reference_outputs: dict) -> bool:
    return int(len(outputs["response"]) < 2 * len(reference_outputs["answer"]))

default_instructions = "以简短、精炼的方式（一个短句）回答用户的问题。"

def my_app(question: str, model: str = "gpt-4.1-mini", instructions: str = default_instructions) -> str:
    return openai_client.chat.completions.create(
        model=model,
        temperature=0,
        messages=[
            {"role": "system", "content": instructions},
            {"role": "user", "content": question},
        ],
    ).choices[0].message.content

def ls_target(inputs: str) -> dict:
    return {"response": my_app(inputs["question"])}

experiment_results = client.evaluate(
    ls_target, # 您的 AI 系统
    data=dataset_name, # 用于预测和评分的数据
    evaluators=[concision, correctness], # 用于评分结果的评估器
    experiment_prefix="openai-4o-mini", # 实验名称的前缀，便于识别
)

def ls_target_v2(inputs: str) -> dict:
    return {"response": my_app(inputs["question"], model="gpt-4-turbo")}

experiment_results = client.evaluate(
    ls_target_v2,
    data=dataset_name,
    evaluators=[concision, correctness],
    experiment_prefix="openai-4-turbo",
)

instructions_v3 = "以简短、精炼的方式（一个短句）回答用户的问题。不要使用超过十个词。"

def ls_target_v3(inputs: str) -> dict:
    response = my_app(
        inputs["question"],
        model="gpt-4-turbo",
        instructions=instructions_v3
    )
    return {"response": response}

experiment_results = client.evaluate(
    ls_target_v3,
    data=dataset_name,
    evaluators=[concision, correctness],
    experiment_prefix="strict-openai-4-turbo",
)

def test_length_score() -> None:
    """测试长度得分至少为 80%。"""
    experiment_results = evaluate(
        ls_target, # 您的 AI 系统
        data=dataset_name, # 用于预测和评分的数据
        evaluators=[concision, correctness], # 用于评分结果的评估器
    )
    # 这将在下一个版本中清理：
    feedback = client.list_feedback(
        run_ids=[r.id for r in client.list_runs(project_name=experiment_results.experiment_name)],
        feedback_key="concision"
    )
    scores = [f.score for f in feedback]
    assert sum(scores) / len(scores) >= 0.8, "总分应至少为 .8"