feat: 新增 LLM checker 支持大模型服务应用层拨测
基于 AI SDK v6 实现 openai/openai-responses/anthropic 三类 provider 的 http/stream 模式调用 支持 output/finishReason/usage/stream 等完整 expect 断言链路 新增 9 个源文件和 5 个测试文件共 78 个测试 更新 README/DEVELOPMENT/probes.example.yaml 和 probe-config.schema.json
This commit is contained in:
@@ -2,6 +2,7 @@ import { CommandChecker } from "./cmd";
|
||||
import { DbChecker } from "./db";
|
||||
import { HttpChecker } from "./http";
|
||||
import { IcmpChecker } from "./icmp";
|
||||
import { LlmChecker } from "./llm";
|
||||
import { CheckerRegistry } from "./registry";
|
||||
import { TcpChecker } from "./tcp";
|
||||
import { UdpChecker } from "./udp";
|
||||
@@ -13,6 +14,7 @@ const checkers = [
|
||||
new TcpChecker(),
|
||||
new IcmpChecker(),
|
||||
new UdpChecker(),
|
||||
new LlmChecker(),
|
||||
];
|
||||
|
||||
export function createDefaultCheckerRegistry(): CheckerRegistry {
|
||||
|
||||
311
src/server/checker/runner/llm/execute.ts
Normal file
311
src/server/checker/runner/llm/execute.ts
Normal file
@@ -0,0 +1,311 @@
|
||||
import type { JSONObject } from "@ai-sdk/provider";
|
||||
|
||||
import { APICallError, generateText, streamText } from "ai";
|
||||
import { isError } from "es-toolkit";
|
||||
|
||||
import type { CheckResult, RawTargetConfig } from "../../types";
|
||||
import type { CheckerContext, CheckerDefinition, CheckerValidationInput, ResolveContext } from "../types";
|
||||
import type { LlmCheckObservation, LlmExpectConfig, LlmTargetConfig, ResolvedLlmTarget } from "./types";
|
||||
|
||||
import { checkDuration } from "../../expect/duration";
|
||||
import { errorFailure } from "../../expect/failure";
|
||||
import { runExpects } from "./expect";
|
||||
import {
|
||||
buildObservationFromApiCallError,
|
||||
buildObservationFromGenerateText,
|
||||
buildObservationFromStreamText,
|
||||
} from "./observation";
|
||||
import { createProviderModel } from "./provider";
|
||||
import { llmCheckerSchemas } from "./schema";
|
||||
import { validateLlmConfig } from "./validate";
|
||||
|
||||
export class LlmChecker implements CheckerDefinition<ResolvedLlmTarget> {
|
||||
readonly configKey = "llm";
|
||||
readonly schemas = llmCheckerSchemas;
|
||||
readonly type = "llm";
|
||||
|
||||
async execute(t: ResolvedLlmTarget, ctx: CheckerContext): Promise<CheckResult> {
|
||||
const timestamp = new Date().toISOString();
|
||||
const expect = t.expect;
|
||||
const start = performance.now();
|
||||
|
||||
try {
|
||||
const { http: httpMeta, model } = createProviderModel(t.llm);
|
||||
|
||||
if (t.llm.mode === "stream") {
|
||||
return await this.executeStream(t, model, httpMeta, expect, ctx, timestamp, start);
|
||||
}
|
||||
|
||||
return await this.executeHttp(t, model, httpMeta, expect, ctx, timestamp, start);
|
||||
} catch (error) {
|
||||
const durationMs = Math.round(performance.now() - start);
|
||||
|
||||
if (error instanceof APICallError) {
|
||||
const observation = buildObservationFromApiCallError(error, t.llm.provider, t.llm.model, t.llm.mode);
|
||||
|
||||
if (observation.http === null) {
|
||||
return {
|
||||
durationMs,
|
||||
failure: errorFailure("request", "request", error.message),
|
||||
matched: false,
|
||||
statusDetail: null,
|
||||
targetId: t.id,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
const durationResult = checkDuration(durationMs, expect?.maxDurationMs);
|
||||
const expectResult = runExpects(observation, expect);
|
||||
const failure = expectResult.failure ?? durationResult.failure;
|
||||
|
||||
return {
|
||||
durationMs,
|
||||
failure,
|
||||
matched: failure === null,
|
||||
statusDetail: buildStatusDetail(observation),
|
||||
targetId: t.id,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
const isTimeout = ctx.signal.aborted || (error instanceof DOMException && error.name === "AbortError");
|
||||
return {
|
||||
durationMs,
|
||||
failure: errorFailure(
|
||||
"request",
|
||||
"request",
|
||||
isTimeout ? `请求超时 (${t.timeoutMs}ms)` : isError(error) ? error.message : String(error),
|
||||
),
|
||||
matched: false,
|
||||
statusDetail: null,
|
||||
targetId: t.id,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
resolve(target: RawTargetConfig, context: ResolveContext): ResolvedLlmTarget {
|
||||
const t = target as RawTargetConfig & { llm: LlmTargetConfig; type: "llm" };
|
||||
const llmDefaults = context.defaults["llm"] as
|
||||
| undefined
|
||||
| {
|
||||
headers?: Record<string, string>;
|
||||
ignoreSSL?: boolean;
|
||||
mode?: string;
|
||||
options?: Record<string, unknown>;
|
||||
providerOptions?: Record<string, Record<string, unknown>>;
|
||||
};
|
||||
|
||||
const resolvedConfig = {
|
||||
authToken: t.llm.authToken,
|
||||
headers: { ...(llmDefaults?.headers ?? {}), ...(t.llm.headers ?? {}) },
|
||||
ignoreSSL: t.llm.ignoreSSL ?? llmDefaults?.ignoreSSL ?? false,
|
||||
key: t.llm.key ?? "",
|
||||
mode: (t.llm.mode ?? llmDefaults?.mode ?? "http") as "http" | "stream",
|
||||
model: t.llm.model,
|
||||
options: {
|
||||
frequencyPenalty:
|
||||
t.llm.options?.frequencyPenalty ?? (llmDefaults?.options?.["frequencyPenalty"] as number | undefined),
|
||||
maxOutputTokens:
|
||||
t.llm.options?.maxOutputTokens ?? (llmDefaults?.options?.["maxOutputTokens"] as number | undefined) ?? 16,
|
||||
presencePenalty:
|
||||
t.llm.options?.presencePenalty ?? (llmDefaults?.options?.["presencePenalty"] as number | undefined),
|
||||
seed: t.llm.options?.seed ?? (llmDefaults?.options?.["seed"] as number | undefined),
|
||||
stopSequences:
|
||||
t.llm.options?.stopSequences ?? (llmDefaults?.options?.["stopSequences"] as string[] | undefined),
|
||||
temperature: t.llm.options?.temperature ?? (llmDefaults?.options?.["temperature"] as number | undefined) ?? 0,
|
||||
topK: t.llm.options?.topK ?? (llmDefaults?.options?.["topK"] as number | undefined),
|
||||
topP: t.llm.options?.topP ?? (llmDefaults?.options?.["topP"] as number | undefined),
|
||||
},
|
||||
prompt: t.llm.prompt,
|
||||
provider: t.llm.provider,
|
||||
providerOptions: {
|
||||
...((llmDefaults?.providerOptions ?? {}) as Record<string, JSONObject>),
|
||||
...(t.llm.providerOptions ?? {}),
|
||||
},
|
||||
url: t.llm.url,
|
||||
};
|
||||
|
||||
return {
|
||||
description: (target.description as null | string) ?? null,
|
||||
expect: target.expect as LlmExpectConfig | undefined,
|
||||
group: target.group ?? "default",
|
||||
id: t.id,
|
||||
intervalMs: context.defaultIntervalMs,
|
||||
llm: resolvedConfig,
|
||||
name: (target.name as null | string) ?? null,
|
||||
timeoutMs: context.defaultTimeoutMs,
|
||||
type: "llm",
|
||||
} satisfies ResolvedLlmTarget;
|
||||
}
|
||||
|
||||
serialize(t: ResolvedLlmTarget): { config: string; target: string } {
|
||||
return {
|
||||
config: JSON.stringify({
|
||||
headers: t.llm.headers,
|
||||
ignoreSSL: t.llm.ignoreSSL,
|
||||
key: t.llm.key ? "***" : "",
|
||||
mode: t.llm.mode,
|
||||
model: t.llm.model,
|
||||
options: t.llm.options,
|
||||
prompt: t.llm.prompt,
|
||||
provider: t.llm.provider,
|
||||
providerOptions: t.llm.providerOptions,
|
||||
url: t.llm.url,
|
||||
}),
|
||||
target: `${t.llm.provider}:${t.llm.model} @ ${t.llm.url}`,
|
||||
};
|
||||
}
|
||||
|
||||
validate(input: CheckerValidationInput) {
|
||||
return validateLlmConfig(input);
|
||||
}
|
||||
|
||||
private async executeHttp(
|
||||
t: ResolvedLlmTarget,
|
||||
model: ReturnType<typeof createProviderModel>["model"],
|
||||
httpMeta: null | { headers: Record<string, string>; status: number; statusText: string },
|
||||
expect: LlmExpectConfig | undefined,
|
||||
ctx: CheckerContext,
|
||||
timestamp: string,
|
||||
start: number,
|
||||
): Promise<CheckResult> {
|
||||
const result = await generateText({
|
||||
abortSignal: ctx.signal,
|
||||
maxRetries: 0,
|
||||
model,
|
||||
prompt: t.llm.prompt,
|
||||
providerOptions: t.llm.providerOptions,
|
||||
...buildSdkOptions(t.llm),
|
||||
});
|
||||
|
||||
const respHeaders = result.response?.headers;
|
||||
const http = httpMeta ?? {
|
||||
headers: respHeaders ? Object.fromEntries(Object.entries(respHeaders)) : {},
|
||||
status: 200,
|
||||
statusText: "",
|
||||
};
|
||||
|
||||
const observation = buildObservationFromGenerateText(
|
||||
t.llm.provider,
|
||||
t.llm.model,
|
||||
t.llm.mode,
|
||||
{
|
||||
finishReason: result.finishReason,
|
||||
rawFinishReason: result.rawFinishReason,
|
||||
text: result.text,
|
||||
usage: {
|
||||
inputTokens: result.usage.inputTokens ?? 0,
|
||||
outputTokens: result.usage.outputTokens ?? 0,
|
||||
totalTokens: result.usage.totalTokens,
|
||||
},
|
||||
warnings: result.warnings?.map((w) =>
|
||||
w.type === "unsupported"
|
||||
? `unsupported: ${w.feature}`
|
||||
: ((w as Record<string, string>)["message"] ?? JSON.stringify(w)),
|
||||
),
|
||||
},
|
||||
http,
|
||||
);
|
||||
|
||||
const durationMs = Math.round(performance.now() - start);
|
||||
const durationResult = checkDuration(durationMs, expect?.maxDurationMs);
|
||||
const expectResult = runExpects(observation, expect);
|
||||
const failure = expectResult.failure ?? durationResult.failure;
|
||||
|
||||
return {
|
||||
durationMs,
|
||||
failure,
|
||||
matched: failure === null,
|
||||
statusDetail: buildStatusDetail(observation),
|
||||
targetId: t.id,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
private async executeStream(
|
||||
t: ResolvedLlmTarget,
|
||||
model: ReturnType<typeof createProviderModel>["model"],
|
||||
httpMeta: null | { headers: Record<string, string>; status: number; statusText: string },
|
||||
expect: LlmExpectConfig | undefined,
|
||||
ctx: CheckerContext,
|
||||
timestamp: string,
|
||||
start: number,
|
||||
): Promise<CheckResult> {
|
||||
const stream = streamText({
|
||||
abortSignal: ctx.signal,
|
||||
maxRetries: 0,
|
||||
model,
|
||||
prompt: t.llm.prompt,
|
||||
providerOptions: t.llm.providerOptions,
|
||||
...buildSdkOptions(t.llm),
|
||||
});
|
||||
|
||||
const observation = await buildObservationFromStreamText(
|
||||
t.llm.provider,
|
||||
t.llm.model,
|
||||
t.llm.mode,
|
||||
stream.fullStream,
|
||||
httpMeta,
|
||||
start,
|
||||
);
|
||||
|
||||
const durationMs = Math.round(performance.now() - start);
|
||||
const durationResult = checkDuration(durationMs, expect?.maxDurationMs);
|
||||
const expectResult = runExpects(observation, expect);
|
||||
const failure = expectResult.failure ?? durationResult.failure;
|
||||
|
||||
return {
|
||||
durationMs,
|
||||
failure,
|
||||
matched: failure === null,
|
||||
statusDetail: buildStatusDetail(observation),
|
||||
targetId: t.id,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function buildSdkOptions(config: ResolvedLlmTarget["llm"]): Record<string, unknown> {
|
||||
const options: Record<string, unknown> = {};
|
||||
const opts = config.options;
|
||||
if (opts.maxOutputTokens !== undefined) options["maxOutputTokens"] = opts.maxOutputTokens;
|
||||
if (opts.temperature !== undefined) options["temperature"] = opts.temperature;
|
||||
if (opts.topP !== undefined) options["topP"] = opts.topP;
|
||||
if (opts.topK !== undefined) options["topK"] = opts.topK;
|
||||
if (opts.presencePenalty !== undefined) options["presencePenalty"] = opts.presencePenalty;
|
||||
if (opts.frequencyPenalty !== undefined) options["frequencyPenalty"] = opts.frequencyPenalty;
|
||||
if (opts.stopSequences !== undefined) options["stopSequences"] = opts.stopSequences;
|
||||
if (opts.seed !== undefined) options["seed"] = opts.seed;
|
||||
return options;
|
||||
}
|
||||
|
||||
function buildStatusDetail(observation: LlmCheckObservation): string {
|
||||
const parts: string[] = [`LLM ${observation.provider} ${observation.mode}`];
|
||||
|
||||
if (observation.http) {
|
||||
parts.push(String(observation.http.status));
|
||||
}
|
||||
|
||||
if (observation.finishReason) {
|
||||
parts.push(`finish=${observation.finishReason}`);
|
||||
}
|
||||
|
||||
if (observation.rawFinishReason) {
|
||||
parts.push(`raw=${observation.rawFinishReason}`);
|
||||
}
|
||||
|
||||
if (observation.stream?.firstTokenMs != null) {
|
||||
parts.push(`firstToken=${observation.stream.firstTokenMs}ms`);
|
||||
}
|
||||
|
||||
if (observation.outputText !== null) {
|
||||
parts.push(`output=${observation.outputText.length} chars`);
|
||||
}
|
||||
|
||||
if (observation.usage) {
|
||||
parts.push(`usage=${observation.usage.inputTokens}/${observation.usage.outputTokens} tokens`);
|
||||
}
|
||||
|
||||
return parts.join(", ");
|
||||
}
|
||||
168
src/server/checker/runner/llm/expect.ts
Normal file
168
src/server/checker/runner/llm/expect.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import type { ExpectResult } from "../../expect/types";
|
||||
import type { LlmCheckObservation, LlmExpectConfig } from "./types";
|
||||
|
||||
import { checkDuration } from "../../expect/duration";
|
||||
import { mismatchFailure } from "../../expect/failure";
|
||||
import { applyOperator } from "../../expect/operator";
|
||||
import { checkHeaders, checkStatus } from "../http/expect";
|
||||
import { checkOutputRules } from "./output";
|
||||
|
||||
export function checkStreamExpect(observation: LlmCheckObservation, expect: LlmExpectConfig): ExpectResult {
|
||||
if (!observation.stream || !expect.stream) return { failure: null, matched: true };
|
||||
|
||||
const expectedCompleted = expect.stream.completed ?? true;
|
||||
if (observation.stream.completed !== expectedCompleted) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"stream",
|
||||
"stream.completed",
|
||||
expectedCompleted,
|
||||
observation.stream.completed,
|
||||
"stream.completed mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (expect.stream.firstTokenMs && observation.stream.firstTokenMs !== null) {
|
||||
if (!applyOperator(observation.stream.firstTokenMs, expect.stream.firstTokenMs)) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"stream",
|
||||
"stream.firstTokenMs",
|
||||
expect.stream.firstTokenMs,
|
||||
observation.stream.firstTokenMs,
|
||||
"stream.firstTokenMs mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
} else if (expect.stream.firstTokenMs && observation.stream.firstTokenMs === null) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"stream",
|
||||
"stream.firstTokenMs",
|
||||
expect.stream.firstTokenMs,
|
||||
null,
|
||||
"stream.firstTokenMs missing",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
export function runExpects(observation: LlmCheckObservation, expect: LlmExpectConfig | undefined): ExpectResult {
|
||||
if (!expect) {
|
||||
const defaultStatus = checkStatus(observation.http?.status ?? 0, [200]);
|
||||
if (!defaultStatus.matched) return defaultStatus;
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
const http = observation.http;
|
||||
|
||||
const statusResult = checkStatus(http?.status ?? 0, expect.status ?? [200]);
|
||||
if (!statusResult.matched) return statusResult;
|
||||
|
||||
if (http && expect.headers) {
|
||||
const headersResult = checkHeaders(http.headers, expect.headers);
|
||||
if (!headersResult.matched) return headersResult;
|
||||
}
|
||||
|
||||
if (observation.stream !== null) {
|
||||
const streamResult = checkStreamExpect(observation, expect);
|
||||
if (!streamResult.matched) return streamResult;
|
||||
}
|
||||
|
||||
const outputResult = checkOutputRules(observation.outputText, expect.output);
|
||||
if (!outputResult.matched) return outputResult;
|
||||
|
||||
if (expect.finishReason !== undefined) {
|
||||
if (observation.finishReason !== expect.finishReason) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"finishReason",
|
||||
"finishReason",
|
||||
expect.finishReason,
|
||||
observation.finishReason,
|
||||
"finishReason mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (expect.rawFinishReason !== undefined) {
|
||||
if (observation.rawFinishReason !== expect.rawFinishReason) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"rawFinishReason",
|
||||
"rawFinishReason",
|
||||
expect.rawFinishReason,
|
||||
observation.rawFinishReason,
|
||||
"rawFinishReason mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (expect.usage && observation.usage) {
|
||||
const usageResult = checkUsageExpect(observation.usage, expect.usage);
|
||||
if (!usageResult.matched) return usageResult;
|
||||
}
|
||||
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
function checkUsageExpect(
|
||||
usage: { inputTokens: number; outputTokens: number; totalTokens: number },
|
||||
expectUsage: { inputTokens?: unknown; outputTokens?: unknown; totalTokens?: unknown },
|
||||
): ExpectResult {
|
||||
if (expectUsage.inputTokens !== undefined) {
|
||||
if (!applyOperator(usage.inputTokens, expectUsage.inputTokens as Parameters<typeof applyOperator>[1])) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"usage",
|
||||
"usage.inputTokens",
|
||||
expectUsage.inputTokens,
|
||||
usage.inputTokens,
|
||||
"usage.inputTokens mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (expectUsage.outputTokens !== undefined) {
|
||||
if (!applyOperator(usage.outputTokens, expectUsage.outputTokens as Parameters<typeof applyOperator>[1])) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"usage",
|
||||
"usage.outputTokens",
|
||||
expectUsage.outputTokens,
|
||||
usage.outputTokens,
|
||||
"usage.outputTokens mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (expectUsage.totalTokens !== undefined) {
|
||||
if (!applyOperator(usage.totalTokens, expectUsage.totalTokens as Parameters<typeof applyOperator>[1])) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"usage",
|
||||
"usage.totalTokens",
|
||||
expectUsage.totalTokens,
|
||||
usage.totalTokens,
|
||||
"usage.totalTokens mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
export { checkDuration };
|
||||
1
src/server/checker/runner/llm/index.ts
Normal file
1
src/server/checker/runner/llm/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { LlmChecker } from "./execute";
|
||||
131
src/server/checker/runner/llm/observation.ts
Normal file
131
src/server/checker/runner/llm/observation.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
import type { APICallError } from "ai";
|
||||
|
||||
import type {
|
||||
LlmCheckObservation,
|
||||
LlmHttpMetadata,
|
||||
LlmMode,
|
||||
LlmProvider,
|
||||
LlmStreamObservation,
|
||||
LlmUsageObservation,
|
||||
} from "./types";
|
||||
|
||||
export function buildObservationFromApiCallError(
|
||||
error: APICallError,
|
||||
provider: LlmProvider,
|
||||
model: string,
|
||||
mode: LlmMode,
|
||||
): LlmCheckObservation {
|
||||
const http: LlmHttpMetadata | null =
|
||||
error.statusCode !== undefined
|
||||
? {
|
||||
headers: error.responseHeaders ?? {},
|
||||
status: error.statusCode,
|
||||
statusText: "",
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
finishReason: null,
|
||||
http,
|
||||
mode,
|
||||
model,
|
||||
outputText: null,
|
||||
provider,
|
||||
rawFinishReason: null,
|
||||
stream: null,
|
||||
usage: null,
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function buildObservationFromGenerateText(
|
||||
provider: LlmProvider,
|
||||
model: string,
|
||||
mode: LlmMode,
|
||||
result: {
|
||||
finishReason: string;
|
||||
rawFinishReason: null | string | undefined;
|
||||
text: string;
|
||||
usage: { inputTokens: number; outputTokens: number; totalTokens?: number | undefined };
|
||||
warnings?: string[];
|
||||
},
|
||||
http: LlmHttpMetadata | null,
|
||||
): LlmCheckObservation {
|
||||
return {
|
||||
finishReason: result.finishReason,
|
||||
http,
|
||||
mode,
|
||||
model,
|
||||
outputText: result.text,
|
||||
provider,
|
||||
rawFinishReason: result.rawFinishReason ?? null,
|
||||
stream: null,
|
||||
usage: {
|
||||
inputTokens: result.usage.inputTokens ?? 0,
|
||||
outputTokens: result.usage.outputTokens ?? 0,
|
||||
totalTokens: result.usage.totalTokens ?? (result.usage.inputTokens ?? 0) + (result.usage.outputTokens ?? 0),
|
||||
},
|
||||
warnings: result.warnings ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildObservationFromStreamText(
|
||||
provider: LlmProvider,
|
||||
model: string,
|
||||
mode: LlmMode,
|
||||
fullStream: AsyncIterable<unknown>,
|
||||
http: LlmHttpMetadata | null,
|
||||
startMs: number,
|
||||
): Promise<LlmCheckObservation> {
|
||||
let outputText = "";
|
||||
let firstTokenMs: null | number = null;
|
||||
let completed = false;
|
||||
let finishReason: null | string = null;
|
||||
let rawFinishReason: null | string = null;
|
||||
let usage: LlmUsageObservation | null = null;
|
||||
const warnings: string[] = [];
|
||||
|
||||
for await (const part of fullStream) {
|
||||
const p = part as Record<string, unknown>;
|
||||
const type = p["type"] as string;
|
||||
|
||||
if (type === "text-delta") {
|
||||
const delta = p["textDelta"] as string;
|
||||
if (delta !== "") {
|
||||
firstTokenMs ??= Math.round(performance.now() - startMs);
|
||||
outputText += delta;
|
||||
}
|
||||
} else if (type === "finish") {
|
||||
completed = true;
|
||||
finishReason = (p["finishReason"] as string) ?? null;
|
||||
rawFinishReason = (p["rawFinishReason"] as string | undefined) ?? null;
|
||||
|
||||
const totalUsage = p["totalUsage"] as
|
||||
| undefined
|
||||
| { inputTokens: number; outputTokens: number; totalTokens: number };
|
||||
const partUsage = p["usage"] as undefined | { inputTokens: number; outputTokens: number };
|
||||
|
||||
usage = {
|
||||
inputTokens: totalUsage?.inputTokens ?? partUsage?.inputTokens ?? 0,
|
||||
outputTokens: totalUsage?.outputTokens ?? partUsage?.outputTokens ?? 0,
|
||||
totalTokens: totalUsage?.totalTokens ?? (partUsage?.inputTokens ?? 0) + (partUsage?.outputTokens ?? 0),
|
||||
};
|
||||
} else if (type === "error") {
|
||||
const err = p["error"] as Error | undefined;
|
||||
warnings.push(err?.message ?? "stream error");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
finishReason,
|
||||
http,
|
||||
mode,
|
||||
model,
|
||||
outputText: outputText || null,
|
||||
provider,
|
||||
rawFinishReason,
|
||||
stream: { completed, firstTokenMs } satisfies LlmStreamObservation,
|
||||
usage,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
83
src/server/checker/runner/llm/output.ts
Normal file
83
src/server/checker/runner/llm/output.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import type { ExpectResult } from "../../expect/types";
|
||||
import type { OutputRule } from "./types";
|
||||
|
||||
import { mismatchFailure } from "../../expect/failure";
|
||||
import { applyOperator, evaluateJsonPath } from "../../expect/operator";
|
||||
|
||||
export function checkOutputRules(outputText: null | string, rules: OutputRule[] | undefined): ExpectResult {
|
||||
if (!rules || rules.length === 0) return { failure: null, matched: true };
|
||||
|
||||
for (const rule of rules) {
|
||||
const result = checkSingleOutputRule(outputText, rule);
|
||||
if (!result.matched) return result;
|
||||
}
|
||||
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
function checkSingleOutputRule(outputText: null | string, rule: OutputRule): ExpectResult {
|
||||
if ("equals" in rule) {
|
||||
if (outputText === null || outputText !== rule.equals) {
|
||||
return {
|
||||
failure: mismatchFailure("output", "output", rule.equals, outputText, "output equals mismatch"),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
if ("contains" in rule) {
|
||||
if (!outputText?.includes(rule.contains)) {
|
||||
return {
|
||||
failure: mismatchFailure(
|
||||
"output",
|
||||
"output",
|
||||
`contains: ${rule.contains}`,
|
||||
outputText,
|
||||
"output contains mismatch",
|
||||
),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
if ("regex" in rule) {
|
||||
if (outputText === null || !new RegExp(rule.regex).test(outputText)) {
|
||||
return {
|
||||
failure: mismatchFailure("output", "output", `match: ${rule.regex}`, outputText, "output regex mismatch"),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
if ("json" in rule) {
|
||||
if (outputText === null) {
|
||||
return {
|
||||
failure: mismatchFailure("output", "output", "valid JSON", null, "output is null, cannot parse JSON"),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(outputText);
|
||||
} catch {
|
||||
return {
|
||||
failure: mismatchFailure("output", "output", "valid JSON", outputText, "output is not valid JSON"),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
|
||||
const value = evaluateJsonPath(parsed, rule.json.path);
|
||||
if (!applyOperator(value, rule.json)) {
|
||||
return {
|
||||
failure: mismatchFailure("output", "output", rule.json, value, "output json mismatch"),
|
||||
matched: false,
|
||||
};
|
||||
}
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
|
||||
return { failure: null, matched: true };
|
||||
}
|
||||
62
src/server/checker/runner/llm/provider.ts
Normal file
62
src/server/checker/runner/llm/provider.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import type { LanguageModel } from "ai";
|
||||
|
||||
import { createAnthropic } from "@ai-sdk/anthropic";
|
||||
import { createOpenAI } from "@ai-sdk/openai";
|
||||
|
||||
import type { LlmHttpMetadata, ResolvedLlmConfig } from "./types";
|
||||
|
||||
export interface ProviderResult {
|
||||
http: LlmHttpMetadata | null;
|
||||
model: LanguageModel;
|
||||
}
|
||||
|
||||
export function createProviderModel(config: ResolvedLlmConfig): ProviderResult {
|
||||
let httpMeta: LlmHttpMetadata | null = null;
|
||||
|
||||
const observingFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
const fetchInit: Record<string, unknown> = { ...init };
|
||||
if (config.ignoreSSL) {
|
||||
fetchInit["tls"] = { rejectUnauthorized: false };
|
||||
}
|
||||
|
||||
const response = await fetch(input, fetchInit);
|
||||
httpMeta = {
|
||||
headers: Object.fromEntries(response.headers),
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
};
|
||||
return response;
|
||||
};
|
||||
|
||||
const sharedOptions = {
|
||||
apiKey: config.key,
|
||||
baseURL: config.url,
|
||||
fetch: observingFetch as typeof fetch,
|
||||
headers: config.headers,
|
||||
};
|
||||
|
||||
let model: LanguageModel;
|
||||
|
||||
switch (config.provider) {
|
||||
case "anthropic": {
|
||||
const provider = createAnthropic({
|
||||
...sharedOptions,
|
||||
...(config.authToken ? { headers: { ...config.headers, Authorization: `Bearer ${config.authToken}` } } : {}),
|
||||
});
|
||||
model = provider.messages(config.model);
|
||||
break;
|
||||
}
|
||||
case "openai": {
|
||||
const provider = createOpenAI(sharedOptions);
|
||||
model = provider.chat(config.model);
|
||||
break;
|
||||
}
|
||||
case "openai-responses": {
|
||||
const provider = createOpenAI(sharedOptions);
|
||||
model = provider.responses(config.model);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { http: httpMeta, model };
|
||||
}
|
||||
115
src/server/checker/runner/llm/schema.ts
Normal file
115
src/server/checker/runner/llm/schema.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { Type } from "@sinclair/typebox";
|
||||
|
||||
import type { CheckerSchemas } from "../types";
|
||||
|
||||
import {
|
||||
createHeaderExpectSchema,
|
||||
createPureOperatorSchema,
|
||||
statusCodePatternSchema,
|
||||
stringMapSchema,
|
||||
} from "../../schema/fragments";
|
||||
|
||||
function createLlmOptionsSchema() {
|
||||
return Type.Object(
|
||||
{
|
||||
frequencyPenalty: Type.Optional(Type.Number()),
|
||||
maxOutputTokens: Type.Optional(Type.Integer({ minimum: 1 })),
|
||||
presencePenalty: Type.Optional(Type.Number()),
|
||||
seed: Type.Optional(Type.Number()),
|
||||
stopSequences: Type.Optional(Type.Array(Type.String())),
|
||||
temperature: Type.Optional(Type.Number()),
|
||||
topK: Type.Optional(Type.Number()),
|
||||
topP: Type.Optional(Type.Number()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
}
|
||||
|
||||
function createLlmOutputRulesSchema() {
|
||||
return Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
contains: Type.Optional(Type.String()),
|
||||
equals: Type.Optional(Type.String()),
|
||||
json: Type.Optional(
|
||||
Type.Object({ path: Type.String(), ...operatorProperties() }, { additionalProperties: false }),
|
||||
),
|
||||
regex: Type.Optional(Type.String()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function operatorProperties() {
|
||||
return {
|
||||
contains: Type.Optional(Type.String()),
|
||||
empty: Type.Optional(Type.Boolean()),
|
||||
equals: Type.Optional(Type.Number()),
|
||||
exists: Type.Optional(Type.Boolean()),
|
||||
gt: Type.Optional(Type.Number()),
|
||||
gte: Type.Optional(Type.Number()),
|
||||
lt: Type.Optional(Type.Number()),
|
||||
lte: Type.Optional(Type.Number()),
|
||||
match: Type.Optional(Type.String()),
|
||||
};
|
||||
}
|
||||
|
||||
export const llmCheckerSchemas: CheckerSchemas = {
|
||||
config: Type.Object(
|
||||
{
|
||||
authToken: Type.Optional(Type.String()),
|
||||
headers: Type.Optional(stringMapSchema),
|
||||
ignoreSSL: Type.Optional(Type.Boolean()),
|
||||
key: Type.Optional(Type.String()),
|
||||
mode: Type.Optional(Type.Union([Type.Literal("http"), Type.Literal("stream")])),
|
||||
model: Type.String({ minLength: 1 }),
|
||||
options: Type.Optional(createLlmOptionsSchema()),
|
||||
prompt: Type.String({ minLength: 1 }),
|
||||
provider: Type.Union([Type.Literal("openai"), Type.Literal("openai-responses"), Type.Literal("anthropic")]),
|
||||
providerOptions: Type.Optional(Type.Record(Type.String(), Type.Object({}, { additionalProperties: true }))),
|
||||
url: Type.String({ minLength: 1 }),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
defaults: Type.Object(
|
||||
{
|
||||
headers: Type.Optional(stringMapSchema),
|
||||
ignoreSSL: Type.Optional(Type.Boolean()),
|
||||
mode: Type.Optional(Type.Union([Type.Literal("http"), Type.Literal("stream")])),
|
||||
options: Type.Optional(createLlmOptionsSchema()),
|
||||
providerOptions: Type.Optional(Type.Record(Type.String(), Type.Object({}, { additionalProperties: true }))),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
expect: Type.Object(
|
||||
{
|
||||
finishReason: Type.Optional(Type.String()),
|
||||
headers: Type.Optional(createHeaderExpectSchema()),
|
||||
maxDurationMs: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
output: Type.Optional(createLlmOutputRulesSchema()),
|
||||
rawFinishReason: Type.Optional(Type.String()),
|
||||
status: Type.Optional(Type.Array(statusCodePatternSchema)),
|
||||
stream: Type.Optional(
|
||||
Type.Object(
|
||||
{
|
||||
completed: Type.Optional(Type.Boolean()),
|
||||
firstTokenMs: Type.Optional(createPureOperatorSchema()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
),
|
||||
usage: Type.Optional(
|
||||
Type.Object(
|
||||
{
|
||||
inputTokens: Type.Optional(createPureOperatorSchema()),
|
||||
outputTokens: Type.Optional(createPureOperatorSchema()),
|
||||
totalTokens: Type.Optional(createPureOperatorSchema()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
};
|
||||
121
src/server/checker/runner/llm/types.ts
Normal file
121
src/server/checker/runner/llm/types.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import type { JSONObject } from "@ai-sdk/provider";
|
||||
|
||||
import type { ExpectOperator, ResolvedTargetBase } from "../../types";
|
||||
|
||||
export interface LlmCheckObservation {
|
||||
finishReason: null | string;
|
||||
http: LlmHttpMetadata | null;
|
||||
mode: LlmMode;
|
||||
model: string;
|
||||
outputText: null | string;
|
||||
provider: LlmProvider;
|
||||
rawFinishReason: null | string;
|
||||
stream: LlmStreamObservation | null;
|
||||
usage: LlmUsageObservation | null;
|
||||
warnings: string[];
|
||||
}
|
||||
export interface LlmDefaultsConfig {
|
||||
headers?: Record<string, string>;
|
||||
ignoreSSL?: boolean;
|
||||
mode?: LlmMode;
|
||||
options?: LlmOptions;
|
||||
providerOptions?: Record<string, JSONObject>;
|
||||
}
|
||||
|
||||
export interface LlmExpectConfig {
|
||||
finishReason?: string;
|
||||
headers?: Record<string, ExpectOperator | string>;
|
||||
maxDurationMs?: number;
|
||||
output?: OutputRule[];
|
||||
rawFinishReason?: string;
|
||||
status?: Array<number | string>;
|
||||
stream?: LlmStreamExpect;
|
||||
usage?: LlmUsageExpect;
|
||||
}
|
||||
|
||||
export interface LlmHttpMetadata {
|
||||
headers: Record<string, string>;
|
||||
status: number;
|
||||
statusText: string;
|
||||
}
|
||||
|
||||
export type LlmMode = "http" | "stream";
|
||||
|
||||
export interface LlmOptions {
|
||||
frequencyPenalty?: number;
|
||||
maxOutputTokens?: number;
|
||||
presencePenalty?: number;
|
||||
seed?: number;
|
||||
stopSequences?: string[];
|
||||
temperature?: number;
|
||||
topK?: number;
|
||||
topP?: number;
|
||||
}
|
||||
|
||||
export type LlmProvider = "anthropic" | "openai" | "openai-responses";
|
||||
|
||||
export interface LlmStreamExpect {
|
||||
completed?: boolean;
|
||||
firstTokenMs?: ExpectOperator;
|
||||
}
|
||||
|
||||
export interface LlmStreamObservation {
|
||||
completed: boolean;
|
||||
firstTokenMs: null | number;
|
||||
}
|
||||
|
||||
export interface LlmTargetConfig {
|
||||
authToken?: string;
|
||||
headers?: Record<string, string>;
|
||||
ignoreSSL?: boolean;
|
||||
key?: string;
|
||||
mode?: LlmMode;
|
||||
model: string;
|
||||
options?: LlmOptions;
|
||||
prompt: string;
|
||||
provider: LlmProvider;
|
||||
providerOptions?: Record<string, JSONObject>;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface LlmUsageExpect {
|
||||
inputTokens?: ExpectOperator;
|
||||
outputTokens?: ExpectOperator;
|
||||
totalTokens?: ExpectOperator;
|
||||
}
|
||||
|
||||
export interface LlmUsageObservation {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export interface OutputJsonRule extends ExpectOperator {
|
||||
path: string;
|
||||
}
|
||||
|
||||
export type OutputRule = { contains: string } | { equals: string } | { json: OutputJsonRule } | { regex: string };
|
||||
|
||||
export interface ResolvedLlmConfig {
|
||||
authToken?: string;
|
||||
headers: Record<string, string>;
|
||||
ignoreSSL: boolean;
|
||||
key: string;
|
||||
mode: LlmMode;
|
||||
model: string;
|
||||
options: LlmOptions;
|
||||
prompt: string;
|
||||
provider: LlmProvider;
|
||||
providerOptions: Record<string, JSONObject>;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface ResolvedLlmTarget extends ResolvedTargetBase {
|
||||
expect?: LlmExpectConfig;
|
||||
group: string;
|
||||
intervalMs: number;
|
||||
llm: ResolvedLlmConfig;
|
||||
name: null | string;
|
||||
timeoutMs: number;
|
||||
type: "llm";
|
||||
}
|
||||
397
src/server/checker/runner/llm/validate.ts
Normal file
397
src/server/checker/runner/llm/validate.ts
Normal file
@@ -0,0 +1,397 @@
|
||||
import { isBoolean, isNumber, isPlainObject, isString } from "es-toolkit";
|
||||
import { isArray } from "es-toolkit/compat";
|
||||
|
||||
import type { ConfigValidationIssue } from "../../schema/issues";
|
||||
import type { CheckerValidationInput } from "../types";
|
||||
|
||||
import { isUnsafeRegex } from "../../expect/redos";
|
||||
import { isPlainRecord, validateOperatorObject } from "../../expect/validate-operator";
|
||||
import { issue, joinPath } from "../../schema/issues";
|
||||
|
||||
const ALLOWED_PROVIDERS = new Set(["anthropic", "openai", "openai-responses"]);
|
||||
const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]);
|
||||
const ALLOWED_MODES = new Set(["http", "stream"]);
|
||||
const OUTPUT_RULE_KEYS = ["contains", "equals", "json", "regex"] as const;
|
||||
|
||||
export function validateLlmConfig(input: CheckerValidationInput): ConfigValidationIssue[] {
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
const defaults =
|
||||
isPlainRecord(input.defaults) && isPlainRecord(input.defaults["llm"]) ? input.defaults["llm"] : undefined;
|
||||
|
||||
if (defaults) {
|
||||
issues.push(...validateLlmDefaults(defaults, "defaults.llm"));
|
||||
}
|
||||
|
||||
for (let i = 0; i < input.targets.length; i++) {
|
||||
const target = input.targets[i] as unknown;
|
||||
if (!isPlainRecord(target)) continue;
|
||||
if (target["type"] !== "llm") continue;
|
||||
issues.push(...validateLlmTarget(target, `targets[${i}]`));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function getTargetName(target: Record<string, unknown>): string | undefined {
|
||||
if (isString(target["name"])) return target["name"];
|
||||
return isString(target["id"]) ? target["id"] : undefined;
|
||||
}
|
||||
|
||||
function isNonNegativeFiniteNumber(value: unknown): boolean {
|
||||
return isNumber(value) && Number.isFinite(value) && value >= 0;
|
||||
}
|
||||
|
||||
function validateLlmDefaults(defaults: Record<string, unknown>, path: string): ConfigValidationIssue[] {
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
if (defaults["mode"] !== undefined && !ALLOWED_MODES.has(defaults["mode"] as string)) {
|
||||
issues.push(issue("invalid-type", joinPath(path, "mode"), "必须为 http 或 stream"));
|
||||
}
|
||||
if (defaults["ignoreSSL"] !== undefined && !isBoolean(defaults["ignoreSSL"])) {
|
||||
issues.push(issue("invalid-type", joinPath(path, "ignoreSSL"), "必须为布尔值"));
|
||||
}
|
||||
if (defaults["headers"] !== undefined) {
|
||||
issues.push(...validateStringMap(defaults["headers"], joinPath(path, "headers")));
|
||||
}
|
||||
if (defaults["options"] !== undefined) {
|
||||
issues.push(...validateLlmOptions(defaults["options"], joinPath(path, "options")));
|
||||
}
|
||||
if (defaults["providerOptions"] !== undefined) {
|
||||
issues.push(...validateProviderOptions(defaults["providerOptions"], joinPath(path, "providerOptions")));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateLlmExpect(
|
||||
target: Record<string, unknown>,
|
||||
path: string,
|
||||
mode: string | undefined,
|
||||
targetName?: string,
|
||||
): ConfigValidationIssue[] {
|
||||
const expect = target["expect"];
|
||||
if (expect === undefined || expect === null || !isPlainRecord(expect)) return [];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
const expectPath = joinPath(path, "expect");
|
||||
|
||||
if (isArray(expect["status"])) {
|
||||
issues.push(...validateStatusValues(expect["status"], joinPath(expectPath, "status"), targetName));
|
||||
}
|
||||
|
||||
if (isPlainRecord(expect["headers"])) {
|
||||
for (const [key, value] of Object.entries(expect["headers"])) {
|
||||
if (isString(value)) continue;
|
||||
issues.push(...validateOperatorObject(value, joinPath(joinPath(expectPath, "headers"), key), targetName));
|
||||
}
|
||||
}
|
||||
|
||||
if (expect["output"] !== undefined) {
|
||||
issues.push(...validateOutputRules(expect["output"], joinPath(expectPath, "output"), targetName));
|
||||
}
|
||||
|
||||
if (expect["finishReason"] !== undefined && !isString(expect["finishReason"])) {
|
||||
issues.push(issue("invalid-type", joinPath(expectPath, "finishReason"), "必须为字符串", targetName));
|
||||
}
|
||||
|
||||
if (expect["rawFinishReason"] !== undefined && !isString(expect["rawFinishReason"])) {
|
||||
issues.push(issue("invalid-type", joinPath(expectPath, "rawFinishReason"), "必须为字符串", targetName));
|
||||
}
|
||||
|
||||
if (expect["usage"] !== undefined) {
|
||||
issues.push(...validateUsageExpect(expect["usage"], joinPath(expectPath, "usage"), targetName));
|
||||
}
|
||||
|
||||
if (expect["stream"] !== undefined) {
|
||||
if (mode === "http") {
|
||||
issues.push(
|
||||
issue("invalid-type", joinPath(expectPath, "stream"), "expect.stream 仅支持 stream mode", targetName),
|
||||
);
|
||||
} else {
|
||||
issues.push(...validateStreamExpect(expect["stream"], joinPath(expectPath, "stream"), targetName));
|
||||
}
|
||||
}
|
||||
|
||||
if (expect["maxDurationMs"] !== undefined && !isNonNegativeFiniteNumber(expect["maxDurationMs"])) {
|
||||
issues.push(issue("invalid-type", joinPath(expectPath, "maxDurationMs"), "必须为非负有限数字", targetName));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateLlmOptions(options: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainRecord(options)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
if (options["maxOutputTokens"] !== undefined) {
|
||||
if (
|
||||
!isNumber(options["maxOutputTokens"]) ||
|
||||
!Number.isInteger(options["maxOutputTokens"]) ||
|
||||
options["maxOutputTokens"] < 1
|
||||
) {
|
||||
issues.push(issue("invalid-type", joinPath(path, "maxOutputTokens"), "必须为正整数", targetName));
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of ["temperature", "topP", "topK", "presencePenalty", "frequencyPenalty", "seed"]) {
|
||||
if (options[key] !== undefined && (!isNumber(options[key]) || !Number.isFinite(options[key]))) {
|
||||
issues.push(issue("invalid-type", joinPath(path, key), "必须为有限数字", targetName));
|
||||
}
|
||||
}
|
||||
|
||||
if (options["stopSequences"] !== undefined) {
|
||||
if (!isArray(options["stopSequences"])) {
|
||||
issues.push(issue("invalid-type", joinPath(path, "stopSequences"), "必须为字符串数组", targetName));
|
||||
} else {
|
||||
for (let i = 0; i < options["stopSequences"].length; i++) {
|
||||
if (!isString(options["stopSequences"][i])) {
|
||||
issues.push(issue("invalid-type", `${joinPath(path, "stopSequences")}[${i}]`, "必须为字符串", targetName));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateLlmTarget(target: Record<string, unknown>, path: string): ConfigValidationIssue[] {
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
const targetName = getTargetName(target);
|
||||
const llm = target["llm"];
|
||||
if (!isPlainRecord(llm)) {
|
||||
issues.push(issue("required", joinPath(path, "llm"), "缺少 llm 配置", targetName));
|
||||
issues.push(...validateLlmExpect(target, path, undefined, targetName));
|
||||
return issues;
|
||||
}
|
||||
|
||||
if (!isString(llm["provider"]) || !ALLOWED_PROVIDERS.has(llm["provider"])) {
|
||||
issues.push(
|
||||
issue(
|
||||
"invalid-type",
|
||||
joinPath(joinPath(path, "llm"), "provider"),
|
||||
"必须为 openai、openai-responses 或 anthropic",
|
||||
targetName,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (!isString(llm["url"]) || llm["url"].trim() === "") {
|
||||
issues.push(issue("required", joinPath(joinPath(path, "llm"), "url"), "缺少 llm.url 字段", targetName));
|
||||
} else {
|
||||
try {
|
||||
const url = new URL(llm["url"]);
|
||||
if (!ALLOWED_PROTOCOLS.has(url.protocol)) {
|
||||
issues.push(
|
||||
issue(
|
||||
"invalid-url",
|
||||
joinPath(joinPath(path, "llm"), "url"),
|
||||
"格式不合法，必须以 http:// 或 https:// 开头",
|
||||
targetName,
|
||||
),
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
issues.push(issue("invalid-url", joinPath(joinPath(path, "llm"), "url"), "格式不合法", targetName));
|
||||
}
|
||||
}
|
||||
|
||||
if (!isString(llm["model"]) || llm["model"].trim() === "") {
|
||||
issues.push(issue("required", joinPath(joinPath(path, "llm"), "model"), "必须为非空字符串", targetName));
|
||||
}
|
||||
|
||||
if (!isString(llm["prompt"]) || llm["prompt"].trim() === "") {
|
||||
issues.push(issue("required", joinPath(joinPath(path, "llm"), "prompt"), "必须为非空字符串", targetName));
|
||||
}
|
||||
|
||||
if (llm["mode"] !== undefined && !ALLOWED_MODES.has(llm["mode"] as string)) {
|
||||
issues.push(issue("invalid-type", joinPath(joinPath(path, "llm"), "mode"), "必须为 http 或 stream", targetName));
|
||||
}
|
||||
|
||||
if (llm["headers"] !== undefined) {
|
||||
issues.push(...validateStringMap(llm["headers"], joinPath(joinPath(path, "llm"), "headers"), targetName));
|
||||
}
|
||||
|
||||
if (llm["ignoreSSL"] !== undefined && !isBoolean(llm["ignoreSSL"])) {
|
||||
issues.push(issue("invalid-type", joinPath(joinPath(path, "llm"), "ignoreSSL"), "必须为布尔值", targetName));
|
||||
}
|
||||
|
||||
const provider = llm["provider"] as string | undefined;
|
||||
|
||||
if (llm["authToken"] !== undefined) {
|
||||
if (provider !== "anthropic") {
|
||||
issues.push(
|
||||
issue(
|
||||
"invalid-auth",
|
||||
joinPath(joinPath(path, "llm"), "authToken"),
|
||||
"authToken 仅支持 anthropic provider",
|
||||
targetName,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
provider === "anthropic" &&
|
||||
isString(llm["key"]) &&
|
||||
llm["key"].trim() !== "" &&
|
||||
isString(llm["authToken"]) &&
|
||||
llm["authToken"].trim() !== ""
|
||||
) {
|
||||
issues.push(
|
||||
issue("auth-conflict", joinPath(joinPath(path, "llm"), "key"), "key 与 authToken 不能同时配置", targetName),
|
||||
);
|
||||
}
|
||||
|
||||
if (llm["options"] !== undefined) {
|
||||
issues.push(...validateLlmOptions(llm["options"], joinPath(joinPath(path, "llm"), "options"), targetName));
|
||||
}
|
||||
|
||||
if (llm["providerOptions"] !== undefined) {
|
||||
issues.push(
|
||||
...validateProviderOptions(
|
||||
llm["providerOptions"],
|
||||
joinPath(joinPath(path, "llm"), "providerOptions"),
|
||||
targetName,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const mode = (llm["mode"] as string | undefined) ?? "http";
|
||||
issues.push(...validateLlmExpect(target, path, mode, targetName));
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateOutputJsonRule(value: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainRecord(value)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
if (!isString(value["path"]) || !value["path"].startsWith("$.") || value["path"].length <= 2) {
|
||||
issues.push(issue("invalid-jsonpath", joinPath(path, "path"), '必须为以 "$." 开头的有效 JSONPath', targetName));
|
||||
}
|
||||
|
||||
const operatorKeys = new Set(["path"]);
|
||||
const operators: Record<string, unknown> = {};
|
||||
for (const [key, val] of Object.entries(value)) {
|
||||
if (operatorKeys.has(key)) continue;
|
||||
operators[key] = val;
|
||||
}
|
||||
issues.push(...validateOperatorObject(operators, path, targetName, { requireAtLeastOne: false }));
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateOutputRegex(value: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isString(value)) return [issue("invalid-type", path, "必须为字符串", targetName)];
|
||||
try {
|
||||
new RegExp(value);
|
||||
} catch {
|
||||
return [issue("invalid-regex", path, "正则不合法", targetName)];
|
||||
}
|
||||
return isUnsafeRegex(value) ? [issue("unsafe-regex", path, "正则存在 ReDoS 风险", targetName)] : [];
|
||||
}
|
||||
|
||||
function validateOutputRules(rules: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isArray(rules)) return [issue("invalid-type", path, "必须为数组", targetName)];
|
||||
return rules.flatMap((rule, index) => validateSingleOutputRule(rule, `${path}[${index}]`, targetName));
|
||||
}
|
||||
|
||||
function validateProviderOptions(value: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainObject(value)) return [issue("invalid-type", path, "必须为 JSON object", targetName)];
|
||||
return [];
|
||||
}
|
||||
|
||||
function validateSingleOutputRule(rule: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainRecord(rule)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
|
||||
const found = OUTPUT_RULE_KEYS.filter((type) => type in rule);
|
||||
if (found.length === 0) return [issue("missing-body-rule", path, "缺少支持的规则类型", targetName)];
|
||||
if (found.length > 1) return [issue("multiple-body-rules", path, "只能配置一种规则类型", targetName)];
|
||||
|
||||
const ruleType = found[0]!;
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
for (const key of Object.keys(rule)) {
|
||||
if (key !== ruleType) issues.push(issue("unknown-field", joinPath(path, key), "是未知字段", targetName));
|
||||
}
|
||||
if (issues.length > 0) return issues;
|
||||
|
||||
switch (ruleType) {
|
||||
case "contains":
|
||||
return isString(rule["contains"])
|
||||
? []
|
||||
: [issue("invalid-type", joinPath(path, "contains"), "必须为字符串", targetName)];
|
||||
case "equals":
|
||||
return isString(rule["equals"])
|
||||
? []
|
||||
: [issue("invalid-type", joinPath(path, "equals"), "必须为字符串", targetName)];
|
||||
case "json":
|
||||
return validateOutputJsonRule(rule["json"], joinPath(path, "json"), targetName);
|
||||
case "regex":
|
||||
return validateOutputRegex(rule["regex"], joinPath(path, "regex"), targetName);
|
||||
}
|
||||
}
|
||||
|
||||
function validateStatusValues(values: unknown[], path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
const value = values[i];
|
||||
const itemPath = `${path}[${i}]`;
|
||||
if (isNumber(value)) {
|
||||
if (!Number.isInteger(value) || value < 100 || value > 599) {
|
||||
issues.push(issue("invalid-status", itemPath, "status 数字必须为 100-599 之间的整数", targetName));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (isString(value)) {
|
||||
if (!/^[1-5]xx$/.test(value)) {
|
||||
issues.push(issue("invalid-status", itemPath, "status 模式必须为 1xx 到 5xx", targetName));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
issues.push(issue("invalid-status", itemPath, "status 必须为整数或 1xx 到 5xx 模式", targetName));
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateStreamExpect(stream: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainRecord(stream)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
if (stream["completed"] !== undefined && !isBoolean(stream["completed"])) {
|
||||
issues.push(issue("invalid-type", joinPath(path, "completed"), "必须为布尔值", targetName));
|
||||
}
|
||||
|
||||
if (stream["firstTokenMs"] !== undefined) {
|
||||
issues.push(...validateOperatorObject(stream["firstTokenMs"], joinPath(path, "firstTokenMs"), targetName));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateStringMap(value: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainObject(value)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (!isString(val)) {
|
||||
issues.push(issue("invalid-type", joinPath(path, key), "必须为字符串", targetName));
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
function validateUsageExpect(usage: unknown, path: string, targetName?: string): ConfigValidationIssue[] {
|
||||
if (!isPlainRecord(usage)) return [issue("invalid-type", path, "必须为对象", targetName)];
|
||||
const issues: ConfigValidationIssue[] = [];
|
||||
|
||||
if (usage["inputTokens"] !== undefined) {
|
||||
issues.push(...validateOperatorObject(usage["inputTokens"], joinPath(path, "inputTokens"), targetName));
|
||||
}
|
||||
if (usage["outputTokens"] !== undefined) {
|
||||
issues.push(...validateOperatorObject(usage["outputTokens"], joinPath(path, "outputTokens"), targetName));
|
||||
}
|
||||
if (usage["totalTokens"] !== undefined) {
|
||||
issues.push(...validateOperatorObject(usage["totalTokens"], joinPath(path, "totalTokens"), targetName));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
Reference in New Issue
Block a user