feat: 新增 LLM checker 支持大模型服务应用层拨测

基于 AI SDK v6 实现 openai/openai-responses/anthropic 三类 provider 的 http/stream 模式调用支持 output/finishReason/usage/stream 等完整 expect 断言链路新增 9 个源文件和 5 个测试文件共 78 个测试更新 README/DEVELOPMENT/probes.example.yaml 和 probe-config.schema.json
2026-05-19 00:06:53 +08:00
parent 52262a31f6
commit 349896bd02
24 changed files with 3511 additions and 8 deletions
--- a/src/server/checker/runner/llm/schema.ts
+++ b/src/server/checker/runner/llm/schema.ts
@@ -0,0 +1,115 @@
+import { Type } from "@sinclair/typebox";
+
+import type { CheckerSchemas } from "../types";
+
+import {
+  createHeaderExpectSchema,
+  createPureOperatorSchema,
+  statusCodePatternSchema,
+  stringMapSchema,
+} from "../../schema/fragments";
+
+function createLlmOptionsSchema() {
+  return Type.Object(
+    {
+      frequencyPenalty: Type.Optional(Type.Number()),
+      maxOutputTokens: Type.Optional(Type.Integer({ minimum: 1 })),
+      presencePenalty: Type.Optional(Type.Number()),
+      seed: Type.Optional(Type.Number()),
+      stopSequences: Type.Optional(Type.Array(Type.String())),
+      temperature: Type.Optional(Type.Number()),
+      topK: Type.Optional(Type.Number()),
+      topP: Type.Optional(Type.Number()),
+    },
+    { additionalProperties: false },
+  );
+}
+
+function createLlmOutputRulesSchema() {
+  return Type.Array(
+    Type.Object(
+      {
+        contains: Type.Optional(Type.String()),
+        equals: Type.Optional(Type.String()),
+        json: Type.Optional(
+          Type.Object({ path: Type.String(), ...operatorProperties() }, { additionalProperties: false }),
+        ),
+        regex: Type.Optional(Type.String()),
+      },
+      { additionalProperties: false },
+    ),
+  );
+}
+
+function operatorProperties() {
+  return {
+    contains: Type.Optional(Type.String()),
+    empty: Type.Optional(Type.Boolean()),
+    equals: Type.Optional(Type.Number()),
+    exists: Type.Optional(Type.Boolean()),
+    gt: Type.Optional(Type.Number()),
+    gte: Type.Optional(Type.Number()),
+    lt: Type.Optional(Type.Number()),
+    lte: Type.Optional(Type.Number()),
+    match: Type.Optional(Type.String()),
+  };
+}
+
+export const llmCheckerSchemas: CheckerSchemas = {
+  config: Type.Object(
+    {
+      authToken: Type.Optional(Type.String()),
+      headers: Type.Optional(stringMapSchema),
+      ignoreSSL: Type.Optional(Type.Boolean()),
+      key: Type.Optional(Type.String()),
+      mode: Type.Optional(Type.Union([Type.Literal("http"), Type.Literal("stream")])),
+      model: Type.String({ minLength: 1 }),
+      options: Type.Optional(createLlmOptionsSchema()),
+      prompt: Type.String({ minLength: 1 }),
+      provider: Type.Union([Type.Literal("openai"), Type.Literal("openai-responses"), Type.Literal("anthropic")]),
+      providerOptions: Type.Optional(Type.Record(Type.String(), Type.Object({}, { additionalProperties: true }))),
+      url: Type.String({ minLength: 1 }),
+    },
+    { additionalProperties: false },
+  ),
+  defaults: Type.Object(
+    {
+      headers: Type.Optional(stringMapSchema),
+      ignoreSSL: Type.Optional(Type.Boolean()),
+      mode: Type.Optional(Type.Union([Type.Literal("http"), Type.Literal("stream")])),
+      options: Type.Optional(createLlmOptionsSchema()),
+      providerOptions: Type.Optional(Type.Record(Type.String(), Type.Object({}, { additionalProperties: true }))),
+    },
+    { additionalProperties: false },
+  ),
+  expect: Type.Object(
+    {
+      finishReason: Type.Optional(Type.String()),
+      headers: Type.Optional(createHeaderExpectSchema()),
+      maxDurationMs: Type.Optional(Type.Number({ minimum: 0 })),
+      output: Type.Optional(createLlmOutputRulesSchema()),
+      rawFinishReason: Type.Optional(Type.String()),
+      status: Type.Optional(Type.Array(statusCodePatternSchema)),
+      stream: Type.Optional(
+        Type.Object(
+          {
+            completed: Type.Optional(Type.Boolean()),
+            firstTokenMs: Type.Optional(createPureOperatorSchema()),
+          },
+          { additionalProperties: false },
+        ),
+      ),
+      usage: Type.Optional(
+        Type.Object(
+          {
+            inputTokens: Type.Optional(createPureOperatorSchema()),
+            outputTokens: Type.Optional(createPureOperatorSchema()),
+            totalTokens: Type.Optional(createPureOperatorSchema()),
+          },
+          { additionalProperties: false },
+        ),
+      ),
+    },
+    { additionalProperties: false },
+  ),
+};