如何评估 LLM 应用

from langsmith import traceable, wrappers
from openai import OpenAI

# 可选：包装 OpenAI 客户端以追踪所有模型调用。
oai_client = wrappers.wrap_openai(OpenAI())

# 可选：添加 'traceable' 装饰器以追踪此函数的输入/输出。
@traceable
def toxicity_classifier(inputs: dict) -> dict:
    instructions = (
      "请审查下面的用户查询，并判断其是否包含任何形式的毒性行为，"
      "例如侮辱、威胁或高度负面评论。如果包含，请回复 'Toxic'；"
      "如果不包含，请回复 'Not toxic'。"
    )
    messages = [
        {"role": "system", "content": instructions},
        {"role": "user", "content": inputs["text"]},
    ]
    result = oai_client.chat.completions.create(
        messages=messages, model="gpt-4.1-mini", temperature=0
    )
    return {"class": result.choices[0].message.content}

import { OpenAI } from "openai";
import { wrapOpenAI } from "langsmith/wrappers";
import { traceable } from "langsmith/traceable";

// 可选：包装 OpenAI 客户端以追踪所有模型调用。
const oaiClient = wrapOpenAI(new OpenAI());

// 可选：添加 'traceable' 包装器以追踪此函数的输入/输出。
const toxicityClassifier = traceable(
  async (text: string) => {
    const result = await oaiClient.chat.completions.create({
      messages: [
        {
           role: "system",
          content: "请审查下面的用户查询，并判断其是否包含任何形式的毒性行为，例如侮辱、威胁或高度负面评论。如果包含，请回复 'Toxic'；如果不包含，请回复 'Not toxic'。",
        },
        { role: "user", content: text },
      ],
      model: "gpt-4.1-mini",
      temperature: 0,
    });

    return result.choices[0].message.content;
  },
  { name: "toxicityClassifier" }
);

from langsmith import Client
ls_client = Client()

examples = [
  {
    "inputs": {"text": "Shut up, idiot"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "You're a wonderful person"},
    "outputs": {"label": "Not toxic"},
  },
  {
    "inputs": {"text": "This is the worst thing ever"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "I had a great day today"},
    "outputs": {"label": "Not toxic"},
  },
  {
    "inputs": {"text": "Nobody likes you"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "This is unacceptable. I want to speak to the manager."},
    "outputs": {"label": "Not toxic"},
  },
]

dataset = ls_client.create_dataset(dataset_name="Toxic Queries")
ls_client.create_examples(
  dataset_id=dataset.id,
  examples=examples,
)

import { Client } from "langsmith";

const langsmith = new Client();

// 创建一个数据集
const labeledTexts = [
  ["Shut up, idiot", "Toxic"],
  ["You're a wonderful person", "Not toxic"],
  ["This is the worst thing ever", "Toxic"],
  ["I had a great day today", "Not toxic"],
  ["Nobody likes you", "Toxic"],
  ["This is unacceptable. I want to speak to the manager.", "Not toxic"],
];

const [inputs, outputs] = labeledTexts.reduce<
  [Array<{ input: string }>, Array<{ outputs: string }>]
>(
  ([inputs, outputs], item) => [
    [...inputs, { input: item[0] }],
    [...outputs, { outputs: item[1] }],
  ],
  [[], []]
);

const datasetName = "Toxic Queries";
const toxicDataset = await langsmith.createDataset(datasetName);
await langsmith.createExamples({ inputs, outputs, datasetId: toxicDataset.id });

def correct(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
    return outputs["class"] == reference_outputs["label"]

import type { EvaluationResult } from "langsmith/evaluation";

function correct({
  outputs,
  referenceOutputs,
}: {
  outputs: Record<string, any>;
  referenceOutputs?: Record<string, any>;
}): EvaluationResult {
  const score = outputs.output === referenceOutputs?.outputs;
  return { key: "correct", score };
}

# 可选的元数据，用于填充 UI 中的模型/提示词/工具列
EXPERIMENT_METADATA = {
    "models": [
        "openai:gpt-4.1-mini",
        {
            "id": ["langchain", "chat_models", "openai", "ChatOpenAI"],
            "lc": 1,
            "type": "constructor",
            "kwargs": {"model_name": "gpt-4.1", "temperature": 0.2},
        },
    ],
    "prompts": ["my-org/my-eval-prompt:abc12345"],
    "tools": [
        {
            "name": "web_search",
            "description": "搜索网络信息",
            "parameters": {
                "type": "object",
                "properties": {"query": {"type": "string"}},
                "required": ["query"],
            },
        },
    ],
}

# 也可以直接使用 'evaluate' 函数：
# from langsmith import evaluate; evaluate(...)
results = ls_client.evaluate(
    toxicity_classifier,
    data=dataset.name,
    evaluators=[correct],
    experiment_prefix="gpt-4.1-mini, baseline",  # 可选，实验名称前缀
    description="Testing the baseline system.",  # 可选，实验描述
    max_concurrency=4,  # 可选，添加并发
    metadata=EXPERIMENT_METADATA,  # 可选，用于填充 UI 中的模型/提示词/工具列
)

import { evaluate } from "langsmith/evaluation";

// 可选的元数据，用于填充 UI 中的模型/提示词/工具列
const EXPERIMENT_METADATA = {
  models: [
    "openai:gpt-4.1-mini",
    {
      id: ["langchain", "chat_models", "openai", "ChatOpenAI"],
      lc: 1,
      type: "constructor",
      kwargs: { model_name: "gpt-4.1", temperature: 0.2 },
    },
  ],
  prompts: ["my-org/my-eval-prompt:abc12345"],
  tools: [
    {
      name: "web_search",
      description: "搜索网络信息",
      parameters: {
        type: "object",
        properties: { query: { type: "string" } },
        required: ["query"],
      },
    },
  ],
};

await evaluate((inputs) => toxicityClassifier(inputs["input"]), {
  data: datasetName,
  evaluators: [correct],
  experimentPrefix: "gpt-4.1-mini, baseline",  // 可选，实验名称前缀
  maxConcurrency: 4, // 可选，添加并发
  metadata: EXPERIMENT_METADATA,  // 可选，用于填充 UI 中的模型/提示词/工具列
});