如何运行成对评估

from langchain_classic import hub
from langchain.chat_models import init_chat_model
from langsmith import evaluate

# 查看提示：https://smith.langchain.com/hub/langchain-ai/pairwise-evaluation-2
prompt = hub.pull("langchain-ai/pairwise-evaluation-2")
model = init_chat_model("gpt-4.1")
chain = prompt | model

def ranked_preference(inputs: dict, outputs: list[dict]) -> list:
    # 假设示例输入有 'question' 键，实验输出有 'answer' 键。
    response = chain.invoke({
        "question": inputs["question"],
        "answer_a": outputs[0].get("answer", "N/A"),
        "answer_b": outputs[1].get("answer", "N/A"),
    })
    if response["Preference"] == 1:
        scores = [1, 0]
    elif response["Preference"] == 2:
        scores = [0, 1]
    else:
        scores = [0, 0]
    return scores

evaluate(
    ("experiment-1", "experiment-2"),  # 替换为您的实验名称/ID
    evaluators=[ranked_preference],
    randomize_order=True,
    max_concurrency=4,
)

import { evaluate} from "langsmith/evaluation";
import { Run } from "langsmith/schemas";
import { wrapOpenAI } from "langsmith/wrappers";
import OpenAI from "openai";
import { z } from "zod";

const openai = wrapOpenAI(new OpenAI());

async function rankedPreference({
  inputs,
  runs,
}: {
  inputs: Record<string, any>;
  runs: Run[];
}) {
  const scores: Record<string, number> = {};
  const [runA, runB] = runs;
  if (!runA || !runB) throw new Error("Expected at least two runs");

  const payload = {
    question: inputs.question,
    answer_a: runA?.outputs?.output ?? "N/A",
    answer_b: runB?.outputs?.output ?? "N/A",
  };

  const output = await openai.chat.completions.create({
    model: "gpt-4-turbo",
    messages: [
      {
        role: "system",
        content: [
          "请扮演公正的裁判，评估两个 AI 助手对下方显示的用户问题提供的回答质量。",
          "您应选择遵循用户指令并更好地回答用户问题的助手。",
          "您的评估应考虑其回答的有用性、相关性、准确性、深度、创意性和详细程度等因素。",
          "通过比较两个回答开始您的评估，并提供简短解释。",
          "避免任何位置偏差，并确保回答的呈现顺序不会影响您的决定。",
          "不要让回答的长度影响您的评估。不要偏袒某些助手的名称。尽可能客观。",
        ].join(" "),
      },
      {
        role: "user",
        content: [
          `[用户问题] ${payload.question}`,
          `[助手 A 回答开始] ${payload.answer_a} [助手 A 回答结束]`,
          `[助手 B 回答开始] ${payload.answer_b} [助手 B 回答结束]`,
        ].join("\n\n"),
      },
    ],
    tool_choice: {
      type: "function",
      function: { name: "Score" },
    },
    tools: [
      {
        type: "function",
        function: {
          name: "Score",
          description: [
            `提供解释后，严格按照以下格式输出您的最终裁决：`,
            `如果助手 A 的回答基于上述因素更好，则输出 "1"。`,
            `如果助手 B 的回答基于上述因素更好，则输出 "2"。`,
            `如果是平局，则输出 "0"。`,
          ].join(" "),
          parameters: {
            type: "object",
            properties: {
              Preference: {
                type: "integer",
                description: "哪个助手的回答更受青睐？",
              },
            },
          },
        },
      },
    ],
  });

  const { Preference } = z
    .object({ Preference: z.number() })
    .parse(
      JSON.parse(output.choices[0].message.tool_calls[0].function.arguments)
    );

  if (Preference === 1) {
    scores[runA.id] = 1;
    scores[runB.id] = 0;
  } else if (Preference === 2) {
    scores[runA.id] = 0;
    scores[runB.id] = 1;
  } else {
    scores[runA.id] = 0;
    scores[runB.id] = 0;
  }

  return { key: "ranked_preference", scores };
}

await evaluate(["earnest-name-40", "reflecting-pump-91"], {
  evaluators: [rankedPreference],
});