import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { ResolvedLlmTarget } from "../../../../../src/server/checker/runner/llm/types"; import type { CheckerContext } from "../../../../../src/server/checker/runner/types"; import { LlmChecker } from "../../../../../src/server/checker/runner/llm/execute"; const MOCK_PORT = 18456; let server: ReturnType; function makeCtx(timeoutMs = 10000): CheckerContext { const controller = new AbortController(); setTimeout(() => controller.abort(), timeoutMs); return { signal: controller.signal }; } function makeTarget( overrides?: Partial, expectOverrides?: Partial, ): ResolvedLlmTarget { return { description: null, expect: expectOverrides, group: "default", id: "test-llm", intervalMs: 30000, llm: { headers: {}, ignoreSSL: false, key: "test-key", mode: "http", model: "gpt-4o-mini", options: { maxOutputTokens: 16, temperature: 0 }, prompt: "Say OK", provider: "openai", providerOptions: {}, url: `http://127.0.0.1:${MOCK_PORT}/v1`, ...overrides, }, name: null, timeoutMs: 10000, type: "llm", }; } function openaiResponse( content: string, options?: { usage?: { completion_tokens: number; prompt_tokens: number; total_tokens: number } }, ) { return JSON.stringify({ choices: [{ finish_reason: "stop", index: 0, message: { content, role: "assistant" } }], created: Date.now(), id: "chatcmpl-test", model: "gpt-4o-mini", object: "chat.completion", usage: options?.usage ?? { completion_tokens: 2, prompt_tokens: 12, total_tokens: 14 }, }); } beforeAll(() => { server = Bun.serve({ fetch(req) { const url = new URL(req.url); const authHeader = req.headers.get("Authorization"); if (url.pathname === "/v1/rate_limit/chat/completions") { return new Response(JSON.stringify({ error: { message: "Rate limit exceeded", type: "rate_limit_error" } }), { headers: { "Content-Type": "application/json" }, status: 429, }); } if (url.pathname === "/v1/server_error/chat/completions") { return new Response(JSON.stringify({ error: { message: "Internal server error", type: "server_error" } }), { headers: { "Content-Type": "application/json" }, status: 500, }); } if (url.pathname === "/v1/no_content/chat/completions") { return new Response( openaiResponse("", { usage: { completion_tokens: 0, prompt_tokens: 5, total_tokens: 5 } }), { headers: { "Content-Type": "application/json" }, status: 200 }, ); } if (authHeader === "Bearer bad-key") { return new Response( JSON.stringify({ error: { message: "Invalid API key", param: null, type: "invalid_request_error" }, }), { headers: { "Content-Type": "application/json" }, status: 401 }, ); } return new Response(openaiResponse("OK"), { headers: { "Content-Type": "application/json" }, status: 200 }); }, port: MOCK_PORT, }); }); afterAll(() => { void server.stop(); }); const checker = new LlmChecker(); describe("LlmChecker execute - 非流式", () => { test("成功调用返回 matched=true", async () => { const result = await checker.execute(makeTarget(), makeCtx()); expect(result.matched).toBe(true); expect(result.failure).toBeNull(); expect(result.statusDetail).toContain("openai"); expect(result.statusDetail).toContain("http"); expect(result.statusDetail).toContain("200"); expect(result.statusDetail).toContain("finish=stop"); }); test("status expect 不匹配", async () => { const result = await checker.execute(makeTarget(undefined, { status: [404] }), makeCtx()); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("status"); }); test("output equals 不匹配", async () => { const result = await checker.execute(makeTarget(undefined, { output: [{ equals: "WRONG" }] }), makeCtx()); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("output"); }); test("output contains 通过", async () => { const result = await checker.execute(makeTarget(undefined, { output: [{ contains: "O" }] }), makeCtx()); expect(result.matched).toBe(true); }); test("finishReason expect 不匹配", async () => { const result = await checker.execute(makeTarget(undefined, { finishReason: { equals: "length" } }), makeCtx()); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("finishReason"); }); test("401 错误可通过 status expect 捕获", async () => { const result = await checker.execute(makeTarget({ key: "bad-key" }, { status: [401] }), makeCtx()); expect(result.matched).toBe(true); }); test("429 错误可通过 status expect 捕获", async () => { const result = await checker.execute( makeTarget({ url: `http://127.0.0.1:${MOCK_PORT}/v1/rate_limit` }, { status: [429] }), makeCtx(), ); expect(result.matched).toBe(true); }); test("500 错误返回 status failure", async () => { const result = await checker.execute( makeTarget({ url: `http://127.0.0.1:${MOCK_PORT}/v1/server_error` }), makeCtx(), ); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("status"); }); test("连接失败返回 request failure", async () => { const result = await checker.execute(makeTarget({ url: "http://127.0.0.1:19999/v1" }), makeCtx(5000)); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("request"); }); test("statusDetail 包含 output 长度和 usage", async () => { const result = await checker.execute(makeTarget(), makeCtx()); expect(result.statusDetail).toContain("output="); expect(result.statusDetail).toContain("chars"); expect(result.statusDetail).toContain("usage="); expect(result.statusDetail).toContain("tokens"); }); test("无文本输出且配置 output expect 失败", async () => { const result = await checker.execute( makeTarget({ url: `http://127.0.0.1:${MOCK_PORT}/v1/no_content` }, { output: [{ equals: "OK" }] }), makeCtx(), ); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("output"); }); test("无 expect 默认 status=200 通过", async () => { const result = await checker.execute(makeTarget(), makeCtx()); expect(result.matched).toBe(true); }); test("headers 断言通过", async () => { const result = await checker.execute( makeTarget(undefined, { headers: { "content-type": "application/json" } }), makeCtx(), ); expect(result.matched).toBe(true); }); test("headers 断言失败", async () => { const result = await checker.execute( makeTarget(undefined, { headers: { "content-type": "text/plain" } }), makeCtx(), ); expect(result.matched).toBe(false); expect(result.failure?.phase).toBe("headers"); }); });