DiAL/tests/server/checker/runner/llm/schema-validate-resolve.test.ts

import { describe, expect, test } from "bun:test";

import type { ResolvedLlmTarget } from "../../../../../src/server/checker/runner/llm/types";
import type { ResolveContext } from "../../../../../src/server/checker/runner/types";
import type { RawTargetConfig } from "../../../../../src/server/checker/types";

import { checkerRegistry } from "../../../../../src/server/checker/runner";
import { validateLlmConfig } from "../../../../../src/server/checker/runner/llm/validate";

interface SerializedConfig {
  headers: Record<string, string>;
  ignoreSSL: boolean;
  key: string;
  mode: string;
  model: string;
  options: Record<string, unknown>;
  prompt: string;
  provider: string;
  providerOptions: Record<string, unknown>;
  url: string;
}

function asLlm(resolved: ReturnType<ReturnType<typeof checkerRegistry.get>["resolve"]>): ResolvedLlmTarget {
  return resolved as ResolvedLlmTarget;
}

function makeRawTarget(overrides?: Partial<RawTargetConfig>): RawTargetConfig {
  return {
    id: "test-llm",
    llm: {
      model: "gpt-4o-mini",
      prompt: "Say OK",
      provider: "openai",
      url: "https://api.openai.com/v1",
    },
    type: "llm",
    ...overrides,
  };
}

function makeResolveContext(overrides?: Partial<ResolveContext>): ResolveContext {
  return {
    configDir: "/tmp",
    defaultIntervalMs: 30000,
    defaultTimeoutMs: 10000,
    ...overrides,
  };
}

function parseSerializedConfig(json: string): SerializedConfig {
  return JSON.parse(json) as SerializedConfig;
}

describe("LlmChecker schema", () => {
  const checker = checkerRegistry.tryGet("llm");

  test("llm checker 注册到 registry", () => {
    expect(checker).toBeDefined();
    expect(checker?.type).toBe("llm");
    expect(checker?.configKey).toBe("llm");
  });

  test("schemas 包含 config、expect", () => {
    expect(checker).toBeDefined();
    expect(Object.keys(checker!.schemas).sort()).toEqual(["config", "expect"].sort());
  });
});

describe("LlmChecker validate", () => {
  test("合法 LLM target 无校验问题", () => {
    const issues = validateLlmConfig({
      targets: [makeRawTarget()],
    });
    expect(issues).toHaveLength(0);
  });

  test("provider 非法报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "m", prompt: "p", provider: "gemini", url: "https://x" },
        }),
      ],
    });
    expect(issues.length).toBeGreaterThan(0);
    expect(issues.some((i) => i.code === "invalid-type" && i.path.includes("provider"))).toBe(true);
  });

  test("url 非法报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "m", prompt: "p", provider: "openai", url: "ftp://bad" },
        }),
      ],
    });
    expect(issues.length).toBeGreaterThan(0);
    expect(issues.some((i) => i.code === "invalid-url")).toBe(true);
  });

  test("model 为空报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.length).toBeGreaterThan(0);
    expect(issues.some((i) => i.path.includes("model"))).toBe(true);
  });

  test("prompt 为空报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "m", prompt: "", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.length).toBeGreaterThan(0);
    expect(issues.some((i) => i.path.includes("prompt"))).toBe(true);
  });

  test("mode 非法报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { mode: "batch", model: "m", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.length).toBeGreaterThan(0);
    expect(issues.some((i) => i.path.includes("mode"))).toBe(true);
  });

  test("openai provider 不允许 authToken", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { authToken: "tok", model: "m", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.code === "invalid-auth")).toBe(true);
  });

  test("anthropic 同时配置 key 和 authToken 报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { authToken: "tok", key: "k", model: "m", prompt: "p", provider: "anthropic", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.code === "auth-conflict")).toBe(true);
  });

  test("ignoreSSL 非布尔值报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { ignoreSSL: "yes", model: "m", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.path.includes("ignoreSSL"))).toBe(true);
  });

  test("options.maxOutputTokens 非正整数报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "m", options: { maxOutputTokens: -1 }, prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.path.includes("maxOutputTokens"))).toBe(true);
  });

  test("options.stopSequences 非字符串数组报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          llm: { model: "m", options: { stopSequences: [123] }, prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.path.includes("stopSequences"))).toBe(true);
  });

  test("expect.output 缺少规则类型报错", () => {
    const issues = validateLlmConfig({
      targets: [makeRawTarget({ expect: { output: [{}] } })],
    });
    expect(issues.some((i) => i.code === "empty-matcher")).toBe(true);
  });

  test("expect.output 直接 matcher 混入 extractor 报错", () => {
    const issues = validateLlmConfig({
      targets: [makeRawTarget({ expect: { output: [{ contains: "y", json: { path: "$.status" } }] } })],
    });
    expect(issues.some((i) => i.code === "invalid-content-expectation")).toBe(true);
  });

  test("expect.output regex ReDoS 报错", () => {
    const issues = validateLlmConfig({
      targets: [makeRawTarget({ expect: { output: [{ regex: "(a+)+" }] } })],
    });
    expect(issues.some((i) => i.code === "unsafe-regex")).toBe(true);
  });

  test("expect.stream 在 mode:http 下报错", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          expect: { stream: { completed: true } },
          llm: { mode: "http", model: "m", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues.some((i) => i.message.includes("stream mode"))).toBe(true);
  });

  test("expect.stream 在 mode:stream 下合法", () => {
    const issues = validateLlmConfig({
      targets: [
        makeRawTarget({
          expect: { stream: { completed: true } },
          llm: { mode: "stream", model: "m", prompt: "p", provider: "openai", url: "https://x" },
        }),
      ],
    });
    expect(issues).toHaveLength(0);
  });
});

describe("LlmChecker resolve", () => {
  const checker = checkerRegistry.tryGet("llm")!;

  test("最简 target 填充默认值", () => {
    const resolved = asLlm(checker.resolve(makeRawTarget(), makeResolveContext()));
    expect(resolved.type).toBe("llm");
    expect(resolved.llm.mode).toBe("http");
    expect(resolved.llm.key).toBe("");
    expect(resolved.llm.ignoreSSL).toBe(false);
    expect(resolved.llm.options.maxOutputTokens).toBe(16);
    expect(resolved.llm.options.temperature).toBe(0);
    expect(resolved.group).toBe("default");
    expect(resolved.intervalMs).toBe(30000);
    expect(resolved.timeoutMs).toBe(10000);
    expect(resolved.rawExpect).toBeUndefined();
    expect(resolved.expect).toEqual({ status: [200] });
  });

  test("stream mode 未配置 expect.stream 时不物化 completed", () => {
    const raw = makeRawTarget({
      expect: { output: [{ contains: "OK" }] },
      llm: {
        mode: "stream",
        model: "gpt-4o-mini",
        prompt: "Say OK",
        provider: "openai",
        url: "https://api.openai.com/v1",
      },
    });

    const resolved = asLlm(checker.resolve(raw, makeResolveContext()));

    expect(resolved.rawExpect).toEqual({ output: [{ contains: "OK" }] });
    expect(resolved.expect?.stream).toBeUndefined();
  });

  test("配置 expect.stream 但省略 completed 时默认 true", () => {
    const raw = makeRawTarget({
      expect: { stream: { firstTokenMs: 100 } },
      llm: {
        mode: "stream",
        model: "gpt-4o-mini",
        prompt: "Say OK",
        provider: "openai",
        url: "https://api.openai.com/v1",
      },
    });

    const resolved = asLlm(checker.resolve(raw, makeResolveContext()));

    expect(resolved.rawExpect).toEqual({ stream: { firstTokenMs: 100 } });
    expect(resolved.expect?.stream).toEqual({ completed: true, firstTokenMs: { equals: 100 } });
  });

  test("serialize 返回正确格式", () => {
    const resolved = asLlm(checker.resolve(makeRawTarget(), makeResolveContext()));
    const serialized = checker.serialize(resolved);
    expect(serialized.target).toBe("openai:gpt-4o-mini @ https://api.openai.com/v1");
    const config = parseSerializedConfig(serialized.config);
    expect(config.provider).toBe("openai");
    expect(config.key).toBe("");
    expect(config.model).toBe("gpt-4o-mini");
  });

  test("serialize 隐藏 key", () => {
    const raw = makeRawTarget({
      llm: { key: "sk-secret-key", model: "m", prompt: "p", provider: "openai", url: "https://x" },
    });
    const resolved = asLlm(checker.resolve(raw, makeResolveContext()));
    const serialized = checker.serialize(resolved);
    const config = parseSerializedConfig(serialized.config);
    expect(config.key).toBe("***");
  });
});