feat: 新增 LLM checker 支持大模型服务应用层拨测
基于 AI SDK v6 实现 openai/openai-responses/anthropic 三类 provider 的 http/stream 模式调用 支持 output/finishReason/usage/stream 等完整 expect 断言链路 新增 9 个源文件和 5 个测试文件共 78 个测试 更新 README/DEVELOPMENT/probes.example.yaml 和 probe-config.schema.json
This commit is contained in:
258
tests/server/checker/runner/llm/provider-observation.test.ts
Normal file
258
tests/server/checker/runner/llm/provider-observation.test.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import {
|
||||
buildObservationFromApiCallError,
|
||||
buildObservationFromGenerateText,
|
||||
buildObservationFromStreamText,
|
||||
} from "../../../../../src/server/checker/runner/llm/observation";
|
||||
import { createProviderModel } from "../../../../../src/server/checker/runner/llm/provider";
|
||||
|
||||
describe("LLM provider factory", () => {
|
||||
test("createProviderModel 返回 model 和 http 初始为 null", () => {
|
||||
const { http, model } = createProviderModel({
|
||||
headers: {},
|
||||
ignoreSSL: false,
|
||||
key: "test-key",
|
||||
mode: "http",
|
||||
model: "gpt-4o-mini",
|
||||
options: {},
|
||||
prompt: "test",
|
||||
provider: "openai",
|
||||
providerOptions: {},
|
||||
url: "https://api.openai.com/v1",
|
||||
});
|
||||
expect(http).toBeNull();
|
||||
expect(model).toBeDefined();
|
||||
});
|
||||
|
||||
test("openai provider 使用 chat 路径", () => {
|
||||
const { model } = createProviderModel({
|
||||
headers: {},
|
||||
ignoreSSL: false,
|
||||
key: "test-key",
|
||||
mode: "http",
|
||||
model: "gpt-4o-mini",
|
||||
options: {},
|
||||
prompt: "test",
|
||||
provider: "openai",
|
||||
providerOptions: {},
|
||||
url: "https://api.openai.com/v1",
|
||||
});
|
||||
expect(model).toBeDefined();
|
||||
});
|
||||
|
||||
test("openai-responses provider 使用 responses 路径", () => {
|
||||
const { model } = createProviderModel({
|
||||
headers: {},
|
||||
ignoreSSL: false,
|
||||
key: "test-key",
|
||||
mode: "http",
|
||||
model: "gpt-4o-mini",
|
||||
options: {},
|
||||
prompt: "test",
|
||||
provider: "openai-responses",
|
||||
providerOptions: {},
|
||||
url: "https://api.openai.com/v1",
|
||||
});
|
||||
expect(model).toBeDefined();
|
||||
});
|
||||
|
||||
test("anthropic provider 使用 messages 路径", () => {
|
||||
const { model } = createProviderModel({
|
||||
headers: {},
|
||||
ignoreSSL: false,
|
||||
key: "test-key",
|
||||
mode: "http",
|
||||
model: "claude-3-5-haiku-20241022",
|
||||
options: {},
|
||||
prompt: "test",
|
||||
provider: "anthropic",
|
||||
providerOptions: {},
|
||||
url: "https://api.anthropic.com/v1",
|
||||
});
|
||||
expect(model).toBeDefined();
|
||||
});
|
||||
|
||||
test("anthropic authToken 映射到 Authorization header", () => {
|
||||
const { model } = createProviderModel({
|
||||
authToken: "my-bearer-token",
|
||||
headers: {},
|
||||
ignoreSSL: false,
|
||||
key: "",
|
||||
mode: "http",
|
||||
model: "claude-3-5-haiku-20241022",
|
||||
options: {},
|
||||
prompt: "test",
|
||||
provider: "anthropic",
|
||||
providerOptions: {},
|
||||
url: "https://api.anthropic.com/v1",
|
||||
});
|
||||
expect(model).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLM observation - generateText", () => {
|
||||
test("构建非流式 observation", () => {
|
||||
const observation = buildObservationFromGenerateText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"http",
|
||||
{
|
||||
finishReason: "stop",
|
||||
rawFinishReason: "stop",
|
||||
text: "OK",
|
||||
usage: { inputTokens: 12, outputTokens: 2, totalTokens: 14 },
|
||||
},
|
||||
{ headers: { "content-type": "application/json" }, status: 200, statusText: "OK" },
|
||||
);
|
||||
|
||||
expect(observation.provider).toBe("openai");
|
||||
expect(observation.model).toBe("gpt-4o-mini");
|
||||
expect(observation.mode).toBe("http");
|
||||
expect(observation.outputText).toBe("OK");
|
||||
expect(observation.finishReason).toBe("stop");
|
||||
expect(observation.rawFinishReason).toBe("stop");
|
||||
expect(observation.usage).toEqual({ inputTokens: 12, outputTokens: 2, totalTokens: 14 });
|
||||
expect(observation.stream).toBeNull();
|
||||
expect(observation.http?.status).toBe(200);
|
||||
});
|
||||
|
||||
test("rawFinishReason 为 undefined 时转为 null", () => {
|
||||
const observation = buildObservationFromGenerateText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"http",
|
||||
{
|
||||
finishReason: "stop",
|
||||
rawFinishReason: undefined,
|
||||
text: "OK",
|
||||
usage: { inputTokens: 5, outputTokens: 1 },
|
||||
},
|
||||
null,
|
||||
);
|
||||
expect(observation.rawFinishReason).toBeNull();
|
||||
});
|
||||
|
||||
test("usage totalTokens 缺失时自动计算", () => {
|
||||
const observation = buildObservationFromGenerateText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"http",
|
||||
{
|
||||
finishReason: "stop",
|
||||
rawFinishReason: "stop",
|
||||
text: "OK",
|
||||
usage: { inputTokens: 10, outputTokens: 3 },
|
||||
},
|
||||
null,
|
||||
);
|
||||
expect(observation.usage?.totalTokens).toBe(13);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLM observation - APICallError", () => {
|
||||
test("带 statusCode 的 APICallError 构建 http metadata", async () => {
|
||||
const { APICallError } = await import("ai");
|
||||
const error = new APICallError({
|
||||
message: "Unauthorized",
|
||||
requestBodyValues: {},
|
||||
responseBody: '{"error":{"message":"Invalid API key"}}',
|
||||
responseHeaders: { "content-type": "application/json" },
|
||||
statusCode: 401,
|
||||
url: "https://api.openai.com/v1/chat/completions",
|
||||
});
|
||||
|
||||
const observation = buildObservationFromApiCallError(error, "openai", "gpt-4o-mini", "http");
|
||||
expect(observation.http?.status).toBe(401);
|
||||
expect(observation.http?.headers).toEqual({ "content-type": "application/json" });
|
||||
expect(observation.outputText).toBeNull();
|
||||
expect(observation.finishReason).toBeNull();
|
||||
expect(observation.usage).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLM observation - streamText", () => {
|
||||
test("消费 fullStream 构建流式 observation", async () => {
|
||||
const parts = [
|
||||
{ textDelta: "Hello", type: "text-delta" },
|
||||
{ textDelta: " world", type: "text-delta" },
|
||||
{
|
||||
finishReason: "stop",
|
||||
rawFinishReason: "stop",
|
||||
totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
type: "finish",
|
||||
usage: { inputTokens: 10, outputTokens: 5 },
|
||||
},
|
||||
];
|
||||
|
||||
async function* fakeStream() {
|
||||
for (const part of parts) {
|
||||
yield await Promise.resolve(part);
|
||||
}
|
||||
}
|
||||
|
||||
const observation = await buildObservationFromStreamText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"stream",
|
||||
fakeStream(),
|
||||
{ headers: {}, status: 200, statusText: "OK" },
|
||||
performance.now() - 100,
|
||||
);
|
||||
|
||||
expect(observation.outputText).toBe("Hello world");
|
||||
expect(observation.stream?.completed).toBe(true);
|
||||
expect(observation.stream?.firstTokenMs).not.toBeNull();
|
||||
expect(observation.finishReason).toBe("stop");
|
||||
expect(observation.rawFinishReason).toBe("stop");
|
||||
expect(observation.usage?.totalTokens).toBe(15);
|
||||
});
|
||||
|
||||
test("空 text-delta 不触发 firstTokenMs", async () => {
|
||||
const parts = [
|
||||
{ textDelta: "", type: "text-delta" },
|
||||
{ textDelta: "OK", type: "text-delta" },
|
||||
{ finishReason: "stop", type: "finish", usage: { inputTokens: 5, outputTokens: 1 } },
|
||||
];
|
||||
|
||||
async function* fakeStream() {
|
||||
for (const part of parts) {
|
||||
yield await Promise.resolve(part);
|
||||
}
|
||||
}
|
||||
|
||||
const observation = await buildObservationFromStreamText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"stream",
|
||||
fakeStream(),
|
||||
null,
|
||||
performance.now(),
|
||||
);
|
||||
expect(observation.stream?.firstTokenMs).not.toBeNull();
|
||||
expect(observation.outputText).toBe("OK");
|
||||
});
|
||||
|
||||
test("error part 添加到 warnings", async () => {
|
||||
const parts = [
|
||||
{ error: new Error("stream broken"), type: "error" },
|
||||
{ finishReason: "error", type: "finish", usage: { inputTokens: 5, outputTokens: 0 } },
|
||||
];
|
||||
|
||||
async function* fakeStream() {
|
||||
for (const part of parts) {
|
||||
yield await Promise.resolve(part);
|
||||
}
|
||||
}
|
||||
|
||||
const observation = await buildObservationFromStreamText(
|
||||
"openai",
|
||||
"gpt-4o-mini",
|
||||
"stream",
|
||||
fakeStream(),
|
||||
null,
|
||||
performance.now(),
|
||||
);
|
||||
expect(observation.warnings).toContain("stream broken");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user