lanyuanxiaoyao/DiAL
1
0
Fork 0
Code Activity

refactor: ProbeEngine 调度引擎重写为 per-target setTimeout 链

Browse Source 
将 per-group setInterval + groupBy 调度模式改为 per-target setTimeout 链，
实现 catch-up 语义（超时后立即补执行）、AbortController 优雅停止、
循环内错误隔离和 overrun warn 日志。
移除 groupBy/probeGroup/timers，新增 sleep/runLoop/runOnce。
新增 croner 依赖供后续 cron 表达式支持使用。
This commit is contained in:
兰缘小妖
2026-05-26 11:35:06 +08:00
parent c120690cf1
commit 08b61cbf47
4 changed files with 329 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
tests/server/checker/engine.test.ts
View File

@@ -50,6 +50,14 @@ function ensureRegistered() {
}
}
function getRunOnce(engine: ProbeEngine) {
return (
engine as unknown as {
runOnce: (t: ResolvedTargetBase) => Promise<Record<string, unknown>>;
}
).runOnce.bind(engine);
}
function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarget>): ResolvedCommandTarget {
return {
cmd: {
@@ -70,6 +78,19 @@ function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarg
};
}
function makeMockResult(targetId: string, overrides?: Partial<Record<string, unknown>>) {
return {
detail: null,
durationMs: 1,
failure: null,
matched: true,
observation: null,
targetId,
timestamp: new Date().toISOString(),
...overrides,
};
}
describe("ProbeEngine", () => {
test("start/stop 不抛错", () => {
ensureRegistered();
@@ -81,15 +102,12 @@ describe("ProbeEngine", () => {
expect(true).toBe(true);
});
test("单次 probeGroup 执行 cmd 检查", async () => {
test("单次 runOnce 执行 cmd 检查", async () => {
const target = makeCommandTarget("cmd-echo");
const mockStore = createMockStore(["cmd-echo"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([target]);
await getRunOnce(engine)(target);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(1);
@@ -119,11 +137,9 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore(["echo-a", "echo-b"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [targetA, targetB]);
const runOnce = getRunOnce(engine);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([targetA, targetB]);
await Promise.all([runOnce(targetA), runOnce(targetB)]);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(2);
@@ -143,11 +159,9 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore(["bad-cmd", "good-cmd"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [badTarget, goodTarget]);
const runOnce = getRunOnce(engine);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([badTarget, goodTarget]);
await Promise.all([runOnce(badTarget), runOnce(goodTarget)]);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(2);
@@ -173,27 +187,28 @@ describe("ProbeEngine", () => {
const goodTarget = makeCommandTarget("good-cmd");
const mockStore = createMockStore(["reject-cmd", "good-cmd"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [rejectTarget, goodTarget]);
const runOnce = getRunOnce(engine);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([rejectTarget, goodTarget]);
await Promise.all([runOnce(rejectTarget), runOnce(goodTarget)]);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(2);
expect(results[0]!["targetId"]).toBe("reject-cmd");
expect(results[0]!["matched"]).toBe(false);
expect(results[0]!["durationMs"]).toBeNull();
expect(results[0]!["observation"]).toBeNull();
expect(results[0]!["failure"]).toEqual({
const rejectResult = results.find((r) => r["targetId"] === "reject-cmd");
const goodResult = results.find((r) => r["targetId"] === "good-cmd");
expect(rejectResult).toBeDefined();
expect(rejectResult!["matched"]).toBe(false);
expect(rejectResult!["durationMs"]).toBeNull();
expect(rejectResult!["observation"]).toBeNull();
expect(rejectResult!["failure"]).toEqual({
kind: "error",
message: "boom",
path: "engine",
phase: "internal",
});
expect(typeof results[0]!["timestamp"]).toBe("string");
expect(results[1]!["targetId"]).toBe("good-cmd");
expect(results[1]!["matched"]).toBe(true);
expect(typeof rejectResult!["timestamp"]).toBe("string");
expect(goodResult).toBeDefined();
expect(goodResult!["matched"]).toBe(true);
checker.execute = originalExecute;
});
@@ -212,11 +227,9 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore(targets.map((t) => t.name ?? t.id)) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, targets, 2);
const runOnce = getRunOnce(engine);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup(targets);
await Promise.all(targets.map((t) => runOnce(t)));
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(5);
@@ -225,7 +238,7 @@ describe("ProbeEngine", () => {
}
});
test("groupByInterval 按间隔分组", () => {
test("不同 interval 的 target 独立调度", () => {
const targetA = makeCommandTarget("a", { intervalMs: 30000 });
const targetB = makeCommandTarget("b", { intervalMs: 30000 });
const targetC = makeCommandTarget("c", { intervalMs: 60000 });
@@ -242,10 +255,7 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore(["other-name"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([target]);
await getRunOnce(engine)(target);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(0);
@@ -281,10 +291,7 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore(["http-test"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [httpTarget]);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([httpTarget]);
await getRunOnce(engine)(httpTarget);
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
expect(results.length).toBe(1);
@@ -365,11 +372,9 @@ describe("ProbeEngine", () => {
0,
logger,
);
const runOnce = getRunOnce(engine);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget("fail-target"), makeCommandTarget("ok-target")]);
await Promise.all([runOnce(makeCommandTarget("fail-target")), runOnce(makeCommandTarget("ok-target"))]);
const errorLogs = logger.entries.filter((e) => e.level === "error");
expect(errorLogs.length).toBeGreaterThanOrEqual(1);
@@ -393,25 +398,17 @@ describe("ProbeEngine", () => {
const originalExecute = checker.execute.bind(checker);
checker.execute = async (target) => {
if (target.id === targetId) {
return {
detail: null,
return makeMockResult(targetId, {
durationMs: 10,
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
matched: false,
observation: null,
targetId,
timestamp: new Date().toISOString(),
};
});
}
return originalExecute(target, { signal: new AbortController().signal });
};
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget(targetId)]);
await getRunOnce(engine)(makeCommandTarget(targetId));
const stateLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("UP → DOWN"));
expect(stateLogs.length).toBe(1);
@@ -432,11 +429,7 @@ describe("ProbeEngine", () => {
} as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget(targetId)]);
await getRunOnce(engine)(makeCommandTarget(targetId));
const recoverLogs = logger.entries.filter((e) => e.level === "info" && e.msg.includes("DOWN → UP"));
expect(recoverLogs.length).toBe(1);
@@ -455,11 +448,7 @@ describe("ProbeEngine", () => {
} as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget(targetId)]);
await getRunOnce(engine)(makeCommandTarget(targetId));
const stateChangeLogs = logger.entries.filter(
(e) => e.msg.includes("UP → DOWN") || e.msg.includes("DOWN → UP") || e.msg.includes("首次 DOWN"),
@@ -477,25 +466,17 @@ describe("ProbeEngine", () => {
const originalExecute = checker.execute.bind(checker);
checker.execute = async (target) => {
if (target.id === targetId) {
return {
detail: null,
return makeMockResult(targetId, {
durationMs: 10,
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
matched: false,
observation: null,
targetId,
timestamp: new Date().toISOString(),
};
});
}
return originalExecute(target, { signal: new AbortController().signal });
};
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget(targetId)]);
await getRunOnce(engine)(makeCommandTarget(targetId));
const firstDownLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("首次 DOWN"));
expect(firstDownLogs.length).toBe(1);
@@ -510,11 +491,7 @@ describe("ProbeEngine", () => {
const mockStore = createMockStore([targetId]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget(targetId)]);
await getRunOnce(engine)(makeCommandTarget(targetId));
const debugLogs = logger.entries.filter((e) => e.level === "debug");
expect(debugLogs.length).toBeGreaterThanOrEqual(1);
@@ -527,11 +504,186 @@ describe("ProbeEngine", () => {
ensureRegistered();
const mockStore = createMockStore(["no-log"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [makeCommandTarget("no-log")]);
await getRunOnce(engine)(makeCommandTarget("no-log"));
});
});
const probeGroup = (
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
).probeGroup.bind(engine);
await probeGroup([makeCommandTarget("no-log")]);
describe("runLoop 调度行为", () => {
test("首次立即执行", async () => {
ensureRegistered();
let callCount = 0;
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = (target) => {
callCount++;
return Promise.resolve(makeMockResult(target.id));
};
const target = makeCommandTarget("immediate", { intervalMs: 60000 });
const mockStore = createMockStore(["immediate"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(20);
expect(callCount).toBeGreaterThanOrEqual(1);
engine.stop();
checker.execute = originalExecute;
});
test("正常调度间隔", async () => {
ensureRegistered();
const callTimes: number[] = [];
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = (target) => {
callTimes.push(performance.now());
return Promise.resolve(makeMockResult(target.id));
};
const target = makeCommandTarget("interval", { intervalMs: 100 });
const mockStore = createMockStore(["interval"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(280);
engine.stop();
expect(callTimes.length).toBeGreaterThanOrEqual(2);
const gap = callTimes[1]! - callTimes[0]!;
expect(gap).toBeGreaterThanOrEqual(80);
expect(gap).toBeLessThan(200);
checker.execute = originalExecute;
});
test("catch-up 语义：超时后立即补执行", async () => {
ensureRegistered();
const callTimes: number[] = [];
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = async (target) => {
callTimes.push(performance.now());
if (callTimes.length === 1) {
await Bun.sleep(150);
}
return makeMockResult(target.id);
};
const target = makeCommandTarget("catchup", { intervalMs: 100 });
const mockStore = createMockStore(["catchup"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(350);
engine.stop();
expect(callTimes.length).toBeGreaterThanOrEqual(2);
const gap = callTimes[1]! - callTimes[0]!;
expect(gap).toBeGreaterThanOrEqual(140);
expect(gap).toBeLessThan(220);
checker.execute = originalExecute;
});
test("overrun warn 日志", async () => {
ensureRegistered();
const logger = createMemoryLogger();
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = async (target) => {
await Bun.sleep(150);
return makeMockResult(target.id, { durationMs: 150 });
};
const target = makeCommandTarget("overrun", { intervalMs: 100 });
const mockStore = createMockStore(["overrun"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target], 20, 0, logger);
engine.start();
await Bun.sleep(250);
engine.stop();
const warnLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("拨测超时"));
expect(warnLogs.length).toBeGreaterThanOrEqual(1);
expect(warnLogs[0]!.obj).toBeDefined();
expect(warnLogs[0]!.obj!["targetId"]).toBe("overrun");
checker.execute = originalExecute;
});
test("无并发重叠：同一 target 不会并发执行", async () => {
ensureRegistered();
let running = 0;
let maxConcurrent = 0;
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = async (target) => {
running++;
maxConcurrent = Math.max(maxConcurrent, running);
await Bun.sleep(60);
running--;
return makeMockResult(target.id, { durationMs: 60 });
};
const target = makeCommandTarget("no-overlap", { intervalMs: 70 });
const mockStore = createMockStore(["no-overlap"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(350);
engine.stop();
expect(maxConcurrent).toBeLessThanOrEqual(1);
checker.execute = originalExecute;
});
test("优雅停止：stop() 后循环快速退出", async () => {
ensureRegistered();
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = (target) => Promise.resolve(makeMockResult(target.id));
const target = makeCommandTarget("graceful", { intervalMs: 60000 });
const mockStore = createMockStore(["graceful"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(20);
const stopStart = performance.now();
engine.stop();
const stopDuration = performance.now() - stopStart;
expect(stopDuration).toBeLessThan(1000);
checker.execute = originalExecute;
});
test("错误隔离：runCheck 抛异常后循环继续", async () => {
ensureRegistered();
let callCount = 0;
const checker = checkerRegistry.get("cmd");
const originalExecute = checker.execute.bind(checker);
checker.execute = (target) => {
callCount++;
if (callCount === 1) {
return Promise.reject(new Error("first fail"));
}
return Promise.resolve(makeMockResult(target.id));
};
const target = makeCommandTarget("error-isolation", { intervalMs: 50 });
const mockStore = createMockStore(["error-isolation"]) as unknown as ProbeStore;
const engine = new ProbeEngine(mockStore, [target]);
engine.start();
await Bun.sleep(180);
engine.stop();
expect(callCount).toBeGreaterThanOrEqual(2);
checker.execute = originalExecute;
});
});
});