Remote evals

import { Levenshtein } from "autoevals";
import { Eval, initDataset, wrapOpenAI } from "braintrust";
import OpenAI from "openai";
import { z } from "zod";
 
const client = wrapOpenAI(new OpenAI({ apiKey: process.env.OPENAI_API_KEY }));
 
Eval("Simple eval", {
  data: initDataset("local dev", { dataset: "sanity" }), // Datasets are currently ignored
  task: async (input, { parameters }) => {
    const completion = await client.chat.completions.create(
      parameters.main.build({
        input: `${parameters.prefix}:${input}`,
      }),
    );
    return completion.choices[0].message.content ?? "";
  },
  // These scores will be used along with any that you configure in the UI
  scores: [Levenshtein],
  parameters: {
    main: {
      type: "prompt",
      name: "Main prompt",
      description: "This is the main prompt",
      default: {
        messages: [
          {
            role: "user",
            content: "{{input}}",
          },
        ],
        model: "gpt-4o",
      },
    },
    another: {
      type: "prompt",
      name: "Another prompt",
      description: "This is another prompt",
      default: {
        messages: [
          {
            role: "user",
            content: "{{input}}",
          },
        ],
        model: "gpt-4o",
      },
    },
    include_prefix: z
      .boolean()
      .default(false)
      .describe("Include a contextual prefix"),
    prefix: z
      .string()
      .describe("The prefix to include")
      .default("this is a math problem"),
    array_of_objects: z
      .array(
        z.object({
          name: z.string(),
          age: z.number(),
        }),
      )
      .default([
        { name: "John", age: 30 },
        { name: "Jane", age: 25 },
      ]),
  },
});