feat: 新增 LLM checker 支持大模型服务应用层拨测
基于 AI SDK v6 实现 openai/openai-responses/anthropic 三类 provider 的 http/stream 模式调用 支持 output/finishReason/usage/stream 等完整 expect 断言链路 新增 9 个源文件和 5 个测试文件共 78 个测试 更新 README/DEVELOPMENT/probes.example.yaml 和 probe-config.schema.json
This commit is contained in:
384
tests/server/checker/runner/llm/schema-validate-resolve.test.ts
Normal file
384
tests/server/checker/runner/llm/schema-validate-resolve.test.ts
Normal file
@@ -0,0 +1,384 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import type { ResolvedLlmTarget } from "../../../../../src/server/checker/runner/llm/types";
|
||||
import type { ResolveContext } from "../../../../../src/server/checker/runner/types";
|
||||
import type { RawTargetConfig } from "../../../../../src/server/checker/types";
|
||||
|
||||
import { checkerRegistry } from "../../../../../src/server/checker/runner";
|
||||
import { validateLlmConfig } from "../../../../../src/server/checker/runner/llm/validate";
|
||||
|
||||
interface SerializedConfig {
|
||||
headers: Record<string, string>;
|
||||
ignoreSSL: boolean;
|
||||
key: string;
|
||||
mode: string;
|
||||
model: string;
|
||||
options: Record<string, unknown>;
|
||||
prompt: string;
|
||||
provider: string;
|
||||
providerOptions: Record<string, unknown>;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function asLlm(resolved: ReturnType<ReturnType<typeof checkerRegistry.get>["resolve"]>): ResolvedLlmTarget {
|
||||
return resolved as ResolvedLlmTarget;
|
||||
}
|
||||
|
||||
function makeRawTarget(overrides?: Partial<RawTargetConfig>): RawTargetConfig {
|
||||
return {
|
||||
id: "test-llm",
|
||||
llm: {
|
||||
model: "gpt-4o-mini",
|
||||
prompt: "Say OK",
|
||||
provider: "openai",
|
||||
url: "https://api.openai.com/v1",
|
||||
},
|
||||
type: "llm",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeResolveContext(overrides?: Partial<ResolveContext>): ResolveContext {
|
||||
return {
|
||||
configDir: "/tmp",
|
||||
defaultIntervalMs: 30000,
|
||||
defaults: {},
|
||||
defaultTimeoutMs: 10000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function parseSerializedConfig(json: string): SerializedConfig {
|
||||
return JSON.parse(json) as SerializedConfig;
|
||||
}
|
||||
|
||||
describe("LlmChecker schema", () => {
|
||||
const checker = checkerRegistry.tryGet("llm");
|
||||
|
||||
test("llm checker 注册到 registry", () => {
|
||||
expect(checker).toBeDefined();
|
||||
expect(checker?.type).toBe("llm");
|
||||
expect(checker?.configKey).toBe("llm");
|
||||
});
|
||||
|
||||
test("schemas 包含 config、defaults、expect", () => {
|
||||
expect(checker).toBeDefined();
|
||||
expect(Object.keys(checker!.schemas).sort()).toEqual(["config", "defaults", "expect"].sort());
|
||||
});
|
||||
});
|
||||
|
||||
describe("LlmChecker validate", () => {
|
||||
test("合法 LLM target 无校验问题", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [makeRawTarget()],
|
||||
});
|
||||
expect(issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("provider 非法报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "m", prompt: "p", provider: "gemini", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.length).toBeGreaterThan(0);
|
||||
expect(issues.some((i) => i.code === "invalid-type" && i.path.includes("provider"))).toBe(true);
|
||||
});
|
||||
|
||||
test("url 非法报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "m", prompt: "p", provider: "openai", url: "ftp://bad" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.length).toBeGreaterThan(0);
|
||||
expect(issues.some((i) => i.code === "invalid-url")).toBe(true);
|
||||
});
|
||||
|
||||
test("model 为空报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.length).toBeGreaterThan(0);
|
||||
expect(issues.some((i) => i.path.includes("model"))).toBe(true);
|
||||
});
|
||||
|
||||
test("prompt 为空报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "m", prompt: "", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.length).toBeGreaterThan(0);
|
||||
expect(issues.some((i) => i.path.includes("prompt"))).toBe(true);
|
||||
});
|
||||
|
||||
test("mode 非法报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { mode: "batch", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.length).toBeGreaterThan(0);
|
||||
expect(issues.some((i) => i.path.includes("mode"))).toBe(true);
|
||||
});
|
||||
|
||||
test("openai provider 不允许 authToken", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { authToken: "tok", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.code === "invalid-auth")).toBe(true);
|
||||
});
|
||||
|
||||
test("anthropic 同时配置 key 和 authToken 报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { authToken: "tok", key: "k", model: "m", prompt: "p", provider: "anthropic", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.code === "auth-conflict")).toBe(true);
|
||||
});
|
||||
|
||||
test("ignoreSSL 非布尔值报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { ignoreSSL: "yes", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.path.includes("ignoreSSL"))).toBe(true);
|
||||
});
|
||||
|
||||
test("options.maxOutputTokens 非正整数报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "m", options: { maxOutputTokens: -1 }, prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.path.includes("maxOutputTokens"))).toBe(true);
|
||||
});
|
||||
|
||||
test("options.stopSequences 非字符串数组报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
llm: { model: "m", options: { stopSequences: [123] }, prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.path.includes("stopSequences"))).toBe(true);
|
||||
});
|
||||
|
||||
test("expect.output 缺少规则类型报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [makeRawTarget({ expect: { output: [{}] } })],
|
||||
});
|
||||
expect(issues.some((i) => i.code === "missing-body-rule")).toBe(true);
|
||||
});
|
||||
|
||||
test("expect.output 同时多种规则类型报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [makeRawTarget({ expect: { output: [{ contains: "y", equals: "x" }] } })],
|
||||
});
|
||||
expect(issues.some((i) => i.code === "multiple-body-rules")).toBe(true);
|
||||
});
|
||||
|
||||
test("expect.output regex ReDoS 报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [makeRawTarget({ expect: { output: [{ regex: "(a+)+" }] } })],
|
||||
});
|
||||
expect(issues.some((i) => i.code === "unsafe-regex")).toBe(true);
|
||||
});
|
||||
|
||||
test("expect.stream 在 mode:http 下报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
expect: { stream: { completed: true } },
|
||||
llm: { mode: "http", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues.some((i) => i.message.includes("stream mode"))).toBe(true);
|
||||
});
|
||||
|
||||
test("expect.stream 在 mode:stream 下合法", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {},
|
||||
targets: [
|
||||
makeRawTarget({
|
||||
expect: { stream: { completed: true } },
|
||||
llm: { mode: "stream", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("defaults.llm 合法配置", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: {
|
||||
llm: { headers: { "X-Custom": "val" }, ignoreSSL: false, mode: "http", options: { maxOutputTokens: 32 } },
|
||||
},
|
||||
targets: [makeRawTarget()],
|
||||
});
|
||||
expect(issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("defaults.llm mode 非法报错", () => {
|
||||
const issues = validateLlmConfig({
|
||||
defaults: { llm: { mode: "batch" } },
|
||||
targets: [makeRawTarget()],
|
||||
});
|
||||
expect(issues.some((i) => i.path.includes("defaults.llm.mode"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LlmChecker resolve", () => {
|
||||
const checker = checkerRegistry.tryGet("llm")!;
|
||||
|
||||
test("最简 target 填充默认值", () => {
|
||||
const resolved = asLlm(checker.resolve(makeRawTarget(), makeResolveContext()));
|
||||
expect(resolved.type).toBe("llm");
|
||||
expect(resolved.llm.mode).toBe("http");
|
||||
expect(resolved.llm.key).toBe("");
|
||||
expect(resolved.llm.ignoreSSL).toBe(false);
|
||||
expect(resolved.llm.options.maxOutputTokens).toBe(16);
|
||||
expect(resolved.llm.options.temperature).toBe(0);
|
||||
expect(resolved.group).toBe("default");
|
||||
expect(resolved.intervalMs).toBe(30000);
|
||||
expect(resolved.timeoutMs).toBe(10000);
|
||||
});
|
||||
|
||||
test("defaults.llm 与 target.llm 浅合并", () => {
|
||||
const raw = makeRawTarget({
|
||||
llm: {
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "gpt-4o-mini",
|
||||
prompt: "Say OK",
|
||||
provider: "openai",
|
||||
url: "https://api.openai.com/v1",
|
||||
},
|
||||
});
|
||||
const ctx = makeResolveContext({
|
||||
defaults: {
|
||||
llm: {
|
||||
headers: { "X-Custom": "default" },
|
||||
ignoreSSL: true,
|
||||
mode: "stream",
|
||||
options: { maxOutputTokens: 64, temperature: 0.5 },
|
||||
},
|
||||
},
|
||||
});
|
||||
const resolved = asLlm(checker.resolve(raw, ctx));
|
||||
expect(resolved.llm.mode).toBe("stream");
|
||||
expect(resolved.llm.ignoreSSL).toBe(true);
|
||||
expect(resolved.llm.headers).toEqual({ Authorization: "Bearer test", "X-Custom": "default" });
|
||||
expect(resolved.llm.options.maxOutputTokens).toBe(64);
|
||||
expect(resolved.llm.options.temperature).toBe(0.5);
|
||||
});
|
||||
|
||||
test("target 字段覆盖 defaults", () => {
|
||||
const raw = makeRawTarget({
|
||||
llm: {
|
||||
ignoreSSL: false,
|
||||
mode: "http",
|
||||
model: "gpt-4o-mini",
|
||||
options: { maxOutputTokens: 8 },
|
||||
prompt: "Say OK",
|
||||
provider: "openai",
|
||||
url: "https://api.openai.com/v1",
|
||||
},
|
||||
});
|
||||
const ctx = makeResolveContext({
|
||||
defaults: {
|
||||
llm: {
|
||||
ignoreSSL: true,
|
||||
mode: "stream",
|
||||
options: { maxOutputTokens: 64 },
|
||||
},
|
||||
},
|
||||
});
|
||||
const resolved = asLlm(checker.resolve(raw, ctx));
|
||||
expect(resolved.llm.mode).toBe("http");
|
||||
expect(resolved.llm.ignoreSSL).toBe(false);
|
||||
expect(resolved.llm.options.maxOutputTokens).toBe(8);
|
||||
});
|
||||
|
||||
test("serialize 返回正确格式", () => {
|
||||
const resolved = asLlm(checker.resolve(makeRawTarget(), makeResolveContext()));
|
||||
const serialized = checker.serialize(resolved);
|
||||
expect(serialized.target).toBe("openai:gpt-4o-mini @ https://api.openai.com/v1");
|
||||
const config = parseSerializedConfig(serialized.config);
|
||||
expect(config.provider).toBe("openai");
|
||||
expect(config.key).toBe("");
|
||||
expect(config.model).toBe("gpt-4o-mini");
|
||||
});
|
||||
|
||||
test("serialize 隐藏 key", () => {
|
||||
const raw = makeRawTarget({
|
||||
llm: { key: "sk-secret-key", model: "m", prompt: "p", provider: "openai", url: "https://x" },
|
||||
});
|
||||
const resolved = asLlm(checker.resolve(raw, makeResolveContext()));
|
||||
const serialized = checker.serialize(resolved);
|
||||
const config = parseSerializedConfig(serialized.config);
|
||||
expect(config.key).toBe("***");
|
||||
});
|
||||
|
||||
test("providerOptions 浅合并", () => {
|
||||
const raw = makeRawTarget({
|
||||
llm: {
|
||||
model: "m",
|
||||
prompt: "p",
|
||||
provider: "openai",
|
||||
providerOptions: { openai: { store: true } },
|
||||
url: "https://x",
|
||||
},
|
||||
});
|
||||
const ctx = makeResolveContext({
|
||||
defaults: {
|
||||
llm: {
|
||||
providerOptions: { openai: { user: "default-user" } },
|
||||
},
|
||||
},
|
||||
});
|
||||
const resolved = asLlm(checker.resolve(raw, ctx));
|
||||
expect(resolved.llm.providerOptions).toEqual({ openai: { store: true } });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user