DiAL/tests/server/checker/runner/llm/provider-observation.test.ts

import { describe, expect, test } from "bun:test";

import {
  buildObservationFromApiCallError,
  buildObservationFromGenerateText,
  buildObservationFromStreamText,
} from "../../../../../src/server/checker/runner/llm/observation";
import { createProviderModel } from "../../../../../src/server/checker/runner/llm/provider";

describe("LLM provider factory", () => {
  test("createProviderModel 返回 model 和 http 初始为 null", () => {
    const { http, model } = createProviderModel({
      headers: {},
      ignoreSSL: false,
      key: "test-key",
      mode: "http",
      model: "gpt-4o-mini",
      options: {},
      prompt: "test",
      provider: "openai",
      providerOptions: {},
      url: "https://api.openai.com/v1",
    });
    expect(http).toBeNull();
    expect(model).toBeDefined();
  });

  test("openai provider 使用 chat 路径", () => {
    const { model } = createProviderModel({
      headers: {},
      ignoreSSL: false,
      key: "test-key",
      mode: "http",
      model: "gpt-4o-mini",
      options: {},
      prompt: "test",
      provider: "openai",
      providerOptions: {},
      url: "https://api.openai.com/v1",
    });
    expect(model).toBeDefined();
  });

  test("openai-responses provider 使用 responses 路径", () => {
    const { model } = createProviderModel({
      headers: {},
      ignoreSSL: false,
      key: "test-key",
      mode: "http",
      model: "gpt-4o-mini",
      options: {},
      prompt: "test",
      provider: "openai-responses",
      providerOptions: {},
      url: "https://api.openai.com/v1",
    });
    expect(model).toBeDefined();
  });

  test("anthropic provider 使用 messages 路径", () => {
    const { model } = createProviderModel({
      headers: {},
      ignoreSSL: false,
      key: "test-key",
      mode: "http",
      model: "claude-3-5-haiku-20241022",
      options: {},
      prompt: "test",
      provider: "anthropic",
      providerOptions: {},
      url: "https://api.anthropic.com/v1",
    });
    expect(model).toBeDefined();
  });

  test("anthropic authToken 映射到 Authorization header", () => {
    const { model } = createProviderModel({
      authToken: "my-bearer-token",
      headers: {},
      ignoreSSL: false,
      key: "",
      mode: "http",
      model: "claude-3-5-haiku-20241022",
      options: {},
      prompt: "test",
      provider: "anthropic",
      providerOptions: {},
      url: "https://api.anthropic.com/v1",
    });
    expect(model).toBeDefined();
  });
});

describe("LLM observation - generateText", () => {
  test("构建非流式 observation", () => {
    const observation = buildObservationFromGenerateText(
      "openai",
      "gpt-4o-mini",
      "http",
      {
        finishReason: "stop",
        rawFinishReason: "stop",
        text: "OK",
        usage: { inputTokens: 12, outputTokens: 2, totalTokens: 14 },
      },
      { headers: { "content-type": "application/json" }, status: 200, statusText: "OK" },
    );

    expect(observation.provider).toBe("openai");
    expect(observation.model).toBe("gpt-4o-mini");
    expect(observation.mode).toBe("http");
    expect(observation.outputText).toBe("OK");
    expect(observation.finishReason).toBe("stop");
    expect(observation.rawFinishReason).toBe("stop");
    expect(observation.usage).toEqual({ inputTokens: 12, outputTokens: 2, totalTokens: 14 });
    expect(observation.stream).toBeNull();
    expect(observation.http?.status).toBe(200);
  });

  test("rawFinishReason 为 undefined 时转为 null", () => {
    const observation = buildObservationFromGenerateText(
      "openai",
      "gpt-4o-mini",
      "http",
      {
        finishReason: "stop",
        rawFinishReason: undefined,
        text: "OK",
        usage: { inputTokens: 5, outputTokens: 1 },
      },
      null,
    );
    expect(observation.rawFinishReason).toBeNull();
  });

  test("usage totalTokens 缺失时自动计算", () => {
    const observation = buildObservationFromGenerateText(
      "openai",
      "gpt-4o-mini",
      "http",
      {
        finishReason: "stop",
        rawFinishReason: "stop",
        text: "OK",
        usage: { inputTokens: 10, outputTokens: 3 },
      },
      null,
    );
    expect(observation.usage?.totalTokens).toBe(13);
  });
});

describe("LLM observation - APICallError", () => {
  test("带 statusCode 的 APICallError 构建 http metadata", async () => {
    const { APICallError } = await import("ai");
    const error = new APICallError({
      message: "Unauthorized",
      requestBodyValues: {},
      responseBody: '{"error":{"message":"Invalid API key"}}',
      responseHeaders: { "content-type": "application/json" },
      statusCode: 401,
      url: "https://api.openai.com/v1/chat/completions",
    });

    const observation = buildObservationFromApiCallError(error, "openai", "gpt-4o-mini", "http");
    expect(observation.http?.status).toBe(401);
    expect(observation.http?.headers).toEqual({ "content-type": "application/json" });
    expect(observation.outputText).toBeNull();
    expect(observation.finishReason).toBeNull();
    expect(observation.usage).toBeNull();
  });
});

describe("LLM observation - streamText", () => {
  test("消费 fullStream 构建流式 observation", async () => {
    const parts = [
      { textDelta: "Hello", type: "text-delta" },
      { textDelta: " world", type: "text-delta" },
      {
        finishReason: "stop",
        rawFinishReason: "stop",
        totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
        type: "finish",
        usage: { inputTokens: 10, outputTokens: 5 },
      },
    ];

    async function* fakeStream() {
      for (const part of parts) {
        yield await Promise.resolve(part);
      }
    }

    const observation = await buildObservationFromStreamText(
      "openai",
      "gpt-4o-mini",
      "stream",
      fakeStream(),
      { headers: {}, status: 200, statusText: "OK" },
      performance.now() - 100,
    );

    expect(observation.outputText).toBe("Hello world");
    expect(observation.stream?.completed).toBe(true);
    expect(observation.stream?.firstTokenMs).not.toBeNull();
    expect(observation.finishReason).toBe("stop");
    expect(observation.rawFinishReason).toBe("stop");
    expect(observation.usage?.totalTokens).toBe(15);
  });

  test("空 text-delta 不触发 firstTokenMs", async () => {
    const parts = [
      { textDelta: "", type: "text-delta" },
      { textDelta: "OK", type: "text-delta" },
      { finishReason: "stop", type: "finish", usage: { inputTokens: 5, outputTokens: 1 } },
    ];

    async function* fakeStream() {
      for (const part of parts) {
        yield await Promise.resolve(part);
      }
    }

    const observation = await buildObservationFromStreamText(
      "openai",
      "gpt-4o-mini",
      "stream",
      fakeStream(),
      null,
      performance.now(),
    );
    expect(observation.stream?.firstTokenMs).not.toBeNull();
    expect(observation.outputText).toBe("OK");
  });

  test("error part 添加到 warnings", async () => {
    const parts = [
      { error: new Error("stream broken"), type: "error" },
      { finishReason: "error", type: "finish", usage: { inputTokens: 5, outputTokens: 0 } },
    ];

    async function* fakeStream() {
      for (const part of parts) {
        yield await Promise.resolve(part);
      }
    }

    const observation = await buildObservationFromStreamText(
      "openai",
      "gpt-4o-mini",
      "stream",
      fakeStream(),
      null,
      performance.now(),
    );
    expect(observation.warnings).toContain("stream broken");
  });
});