refactor: ProbeEngine 调度引擎重写为 per-target setTimeout 链
将 per-group setInterval + groupBy 调度模式改为 per-target setTimeout 链, 实现 catch-up 语义(超时后立即补执行)、AbortController 优雅停止、 循环内错误隔离和 overrun warn 日志。 移除 groupBy/probeGroup/timers,新增 sleep/runLoop/runOnce。 新增 croner 依赖供后续 cron 表达式支持使用。
This commit is contained in:
3
bun.lock
3
bun.lock
@@ -14,6 +14,7 @@
|
||||
"ai": "^6",
|
||||
"ajv": "^8.20.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"croner": "^10.0.1",
|
||||
"es-toolkit": "^1.46.1",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
@@ -497,6 +498,8 @@
|
||||
|
||||
"cosmiconfig-typescript-loader": ["cosmiconfig-typescript-loader@6.3.0", "https://registry.npmmirror.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.3.0.tgz", { "dependencies": { "jiti": "2.6.1" }, "peerDependencies": { "@types/node": "*", "cosmiconfig": ">=9", "typescript": ">=5" } }, "sha512-Akr82WH1Wfqatyiqpj8HDkO2o2KmJRu1FhKfSNJP3K4IdXwHfEyL7MOb62i1AGQVLtIQM+iCE9CGOtrfhR+mmA=="],
|
||||
|
||||
"croner": ["croner@10.0.1", "", {}, "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g=="],
|
||||
|
||||
"cross-spawn": ["cross-spawn@7.0.6", "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.6.tgz", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "https://registry.npmmirror.com/css-select/-/css-select-5.2.2.tgz", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"ai": "^6",
|
||||
"ajv": "^8.20.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"croner": "^10.0.1",
|
||||
"es-toolkit": "^1.46.1",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { groupBy, isError, Semaphore } from "es-toolkit";
|
||||
import { isError, Semaphore } from "es-toolkit";
|
||||
|
||||
import type { Logger } from "../logger";
|
||||
import type { ProbeStore } from "./store";
|
||||
@@ -11,14 +11,15 @@ import { checkerRegistry } from "./runner";
|
||||
const PRUNE_INTERVAL_MS = 3600000;
|
||||
|
||||
export class ProbeEngine {
|
||||
private abort: AbortController | null = null;
|
||||
private lastMatched = new Map<string, boolean>();
|
||||
private logger: Logger;
|
||||
private pruneTimer: null | ReturnType<typeof setInterval> = null;
|
||||
private retentionMs: number;
|
||||
private semaphore: Semaphore;
|
||||
private store: ProbeStore;
|
||||
private targetIds = new Set<string>();
|
||||
private targets: ResolvedTargetBase[];
|
||||
private timers: Array<ReturnType<typeof setInterval>> = [];
|
||||
|
||||
constructor(
|
||||
store: ProbeStore,
|
||||
@@ -37,32 +38,28 @@ export class ProbeEngine {
|
||||
}
|
||||
|
||||
start(): void {
|
||||
const groups = groupBy(this.targets, (t) => t.intervalMs);
|
||||
this.abort = new AbortController();
|
||||
const signal = this.abort.signal;
|
||||
|
||||
for (const [intervalMs, groupTargets] of Object.entries(groups)) {
|
||||
void this.probeGroup(groupTargets);
|
||||
|
||||
const timer = setInterval(() => {
|
||||
void this.probeGroup(groupTargets);
|
||||
}, Number(intervalMs));
|
||||
|
||||
this.timers.push(timer);
|
||||
for (const target of this.targets) {
|
||||
void this.runLoop(target, signal);
|
||||
}
|
||||
|
||||
if (this.retentionMs > 0) {
|
||||
this.store.prune(this.retentionMs);
|
||||
const pruneTimer = setInterval(() => {
|
||||
this.pruneTimer = setInterval(() => {
|
||||
this.store.prune(this.retentionMs);
|
||||
}, PRUNE_INTERVAL_MS);
|
||||
this.timers.push(pruneTimer);
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
for (const timer of this.timers) {
|
||||
clearInterval(timer);
|
||||
this.abort?.abort();
|
||||
this.abort = null;
|
||||
if (this.pruneTimer) {
|
||||
clearInterval(this.pruneTimer);
|
||||
this.pruneTimer = null;
|
||||
}
|
||||
this.timers = [];
|
||||
}
|
||||
|
||||
private initStateCache(): void {
|
||||
@@ -108,44 +105,6 @@ export class ProbeEngine {
|
||||
this.lastMatched.set(result.targetId, current);
|
||||
}
|
||||
|
||||
private async probeGroup(targets: ResolvedTargetBase[]): Promise<void> {
|
||||
const results = await Promise.allSettled(
|
||||
targets.map(async (target) => {
|
||||
await this.semaphore.acquire();
|
||||
try {
|
||||
return await this.runCheck(target);
|
||||
} finally {
|
||||
this.semaphore.release();
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
for (const [index, result] of results.entries()) {
|
||||
if (result.status === "fulfilled") {
|
||||
this.writeResult(result.value);
|
||||
this.logStateChange(result.value);
|
||||
this.logCheckDebug(result.value);
|
||||
} else {
|
||||
const target = targets[index];
|
||||
if (target) {
|
||||
this.logger.error(
|
||||
{ reason: formatReason(result.reason), targetId: target.id, targetType: target.type },
|
||||
`探针执行失败: ${formatReason(result.reason)}`,
|
||||
);
|
||||
this.writeResult({
|
||||
detail: null,
|
||||
durationMs: null,
|
||||
failure: errorFailure("internal", "engine", formatReason(result.reason)),
|
||||
matched: false,
|
||||
observation: null,
|
||||
targetId: target.id,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private refreshCache(): void {
|
||||
this.targetIds.clear();
|
||||
for (const target of this.store.getTargets()) {
|
||||
@@ -165,6 +124,62 @@ export class ProbeEngine {
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(target: ResolvedTargetBase, signal: AbortSignal): Promise<void> {
|
||||
while (!signal.aborted) {
|
||||
const start = performance.now();
|
||||
try {
|
||||
await this.runOnce(target, signal);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
|
||||
const elapsed = performance.now() - start;
|
||||
if (elapsed > target.intervalMs) {
|
||||
this.logger.warn(
|
||||
{ elapsed, intervalMs: target.intervalMs, targetId: target.id },
|
||||
`拨测超时: ${target.id} 耗时 ${Math.round(elapsed)}ms > 间隔 ${target.intervalMs}ms`,
|
||||
);
|
||||
}
|
||||
const delay = Math.max(0, target.intervalMs - elapsed);
|
||||
try {
|
||||
await sleep(delay, signal);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runOnce(target: ResolvedTargetBase, signal?: AbortSignal): Promise<CheckResult> {
|
||||
await this.semaphore.acquire();
|
||||
if (signal?.aborted) {
|
||||
this.semaphore.release();
|
||||
throw new DOMException("Aborted", "AbortError");
|
||||
}
|
||||
try {
|
||||
const result = await this.runCheck(target);
|
||||
this.writeResult(result);
|
||||
this.logStateChange(result);
|
||||
this.logCheckDebug(result);
|
||||
return result;
|
||||
} catch (error) {
|
||||
const reason = formatReason(error);
|
||||
this.logger.error({ reason, targetId: target.id, targetType: target.type }, `探针执行失败: ${reason}`);
|
||||
const errorResult: CheckResult = {
|
||||
detail: null,
|
||||
durationMs: null,
|
||||
failure: errorFailure("internal", "engine", reason),
|
||||
matched: false,
|
||||
observation: null,
|
||||
targetId: target.id,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
this.writeResult(errorResult);
|
||||
return errorResult;
|
||||
} finally {
|
||||
this.semaphore.release();
|
||||
}
|
||||
}
|
||||
|
||||
private writeResult(result: CheckResult): void {
|
||||
if (!this.targetIds.has(result.targetId)) return;
|
||||
|
||||
@@ -182,3 +197,24 @@ export class ProbeEngine {
|
||||
function formatReason(reason: unknown): string {
|
||||
return isError(reason) ? reason.message : String(reason);
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (signal.aborted) {
|
||||
reject(new DOMException("Aborted", "AbortError"));
|
||||
return;
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
|
||||
function onAbort() {
|
||||
clearTimeout(timer);
|
||||
reject(new DOMException("Aborted", "AbortError"));
|
||||
}
|
||||
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -50,6 +50,14 @@ function ensureRegistered() {
|
||||
}
|
||||
}
|
||||
|
||||
function getRunOnce(engine: ProbeEngine) {
|
||||
return (
|
||||
engine as unknown as {
|
||||
runOnce: (t: ResolvedTargetBase) => Promise<Record<string, unknown>>;
|
||||
}
|
||||
).runOnce.bind(engine);
|
||||
}
|
||||
|
||||
function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarget>): ResolvedCommandTarget {
|
||||
return {
|
||||
cmd: {
|
||||
@@ -70,6 +78,19 @@ function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarg
|
||||
};
|
||||
}
|
||||
|
||||
function makeMockResult(targetId: string, overrides?: Partial<Record<string, unknown>>) {
|
||||
return {
|
||||
detail: null,
|
||||
durationMs: 1,
|
||||
failure: null,
|
||||
matched: true,
|
||||
observation: null,
|
||||
targetId,
|
||||
timestamp: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("ProbeEngine", () => {
|
||||
test("start/stop 不抛错", () => {
|
||||
ensureRegistered();
|
||||
@@ -81,15 +102,12 @@ describe("ProbeEngine", () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
|
||||
test("单次 probeGroup 执行 cmd 检查", async () => {
|
||||
test("单次 runOnce 执行 cmd 检查", async () => {
|
||||
const target = makeCommandTarget("cmd-echo");
|
||||
const mockStore = createMockStore(["cmd-echo"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([target]);
|
||||
await getRunOnce(engine)(target);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(1);
|
||||
@@ -119,11 +137,9 @@ describe("ProbeEngine", () => {
|
||||
|
||||
const mockStore = createMockStore(["echo-a", "echo-b"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [targetA, targetB]);
|
||||
const runOnce = getRunOnce(engine);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([targetA, targetB]);
|
||||
await Promise.all([runOnce(targetA), runOnce(targetB)]);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(2);
|
||||
@@ -143,11 +159,9 @@ describe("ProbeEngine", () => {
|
||||
|
||||
const mockStore = createMockStore(["bad-cmd", "good-cmd"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [badTarget, goodTarget]);
|
||||
const runOnce = getRunOnce(engine);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([badTarget, goodTarget]);
|
||||
await Promise.all([runOnce(badTarget), runOnce(goodTarget)]);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(2);
|
||||
@@ -173,27 +187,28 @@ describe("ProbeEngine", () => {
|
||||
const goodTarget = makeCommandTarget("good-cmd");
|
||||
const mockStore = createMockStore(["reject-cmd", "good-cmd"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [rejectTarget, goodTarget]);
|
||||
const runOnce = getRunOnce(engine);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([rejectTarget, goodTarget]);
|
||||
await Promise.all([runOnce(rejectTarget), runOnce(goodTarget)]);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(2);
|
||||
expect(results[0]!["targetId"]).toBe("reject-cmd");
|
||||
expect(results[0]!["matched"]).toBe(false);
|
||||
expect(results[0]!["durationMs"]).toBeNull();
|
||||
expect(results[0]!["observation"]).toBeNull();
|
||||
expect(results[0]!["failure"]).toEqual({
|
||||
|
||||
const rejectResult = results.find((r) => r["targetId"] === "reject-cmd");
|
||||
const goodResult = results.find((r) => r["targetId"] === "good-cmd");
|
||||
expect(rejectResult).toBeDefined();
|
||||
expect(rejectResult!["matched"]).toBe(false);
|
||||
expect(rejectResult!["durationMs"]).toBeNull();
|
||||
expect(rejectResult!["observation"]).toBeNull();
|
||||
expect(rejectResult!["failure"]).toEqual({
|
||||
kind: "error",
|
||||
message: "boom",
|
||||
path: "engine",
|
||||
phase: "internal",
|
||||
});
|
||||
expect(typeof results[0]!["timestamp"]).toBe("string");
|
||||
expect(results[1]!["targetId"]).toBe("good-cmd");
|
||||
expect(results[1]!["matched"]).toBe(true);
|
||||
expect(typeof rejectResult!["timestamp"]).toBe("string");
|
||||
expect(goodResult).toBeDefined();
|
||||
expect(goodResult!["matched"]).toBe(true);
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
@@ -212,11 +227,9 @@ describe("ProbeEngine", () => {
|
||||
|
||||
const mockStore = createMockStore(targets.map((t) => t.name ?? t.id)) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, targets, 2);
|
||||
const runOnce = getRunOnce(engine);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup(targets);
|
||||
await Promise.all(targets.map((t) => runOnce(t)));
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(5);
|
||||
@@ -225,7 +238,7 @@ describe("ProbeEngine", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("groupByInterval 按间隔分组", () => {
|
||||
test("不同 interval 的 target 独立调度", () => {
|
||||
const targetA = makeCommandTarget("a", { intervalMs: 30000 });
|
||||
const targetB = makeCommandTarget("b", { intervalMs: 30000 });
|
||||
const targetC = makeCommandTarget("c", { intervalMs: 60000 });
|
||||
@@ -242,10 +255,7 @@ describe("ProbeEngine", () => {
|
||||
const mockStore = createMockStore(["other-name"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([target]);
|
||||
await getRunOnce(engine)(target);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(0);
|
||||
@@ -281,10 +291,7 @@ describe("ProbeEngine", () => {
|
||||
const mockStore = createMockStore(["http-test"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [httpTarget]);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([httpTarget]);
|
||||
await getRunOnce(engine)(httpTarget);
|
||||
|
||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||
expect(results.length).toBe(1);
|
||||
@@ -365,11 +372,9 @@ describe("ProbeEngine", () => {
|
||||
0,
|
||||
logger,
|
||||
);
|
||||
const runOnce = getRunOnce(engine);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget("fail-target"), makeCommandTarget("ok-target")]);
|
||||
await Promise.all([runOnce(makeCommandTarget("fail-target")), runOnce(makeCommandTarget("ok-target"))]);
|
||||
|
||||
const errorLogs = logger.entries.filter((e) => e.level === "error");
|
||||
expect(errorLogs.length).toBeGreaterThanOrEqual(1);
|
||||
@@ -393,25 +398,17 @@ describe("ProbeEngine", () => {
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = async (target) => {
|
||||
if (target.id === targetId) {
|
||||
return {
|
||||
detail: null,
|
||||
return makeMockResult(targetId, {
|
||||
durationMs: 10,
|
||||
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
||||
matched: false,
|
||||
observation: null,
|
||||
targetId,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
}
|
||||
return originalExecute(target, { signal: new AbortController().signal });
|
||||
};
|
||||
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget(targetId)]);
|
||||
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||
|
||||
const stateLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("UP → DOWN"));
|
||||
expect(stateLogs.length).toBe(1);
|
||||
@@ -432,11 +429,7 @@ describe("ProbeEngine", () => {
|
||||
} as unknown as ProbeStore;
|
||||
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget(targetId)]);
|
||||
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||
|
||||
const recoverLogs = logger.entries.filter((e) => e.level === "info" && e.msg.includes("DOWN → UP"));
|
||||
expect(recoverLogs.length).toBe(1);
|
||||
@@ -455,11 +448,7 @@ describe("ProbeEngine", () => {
|
||||
} as unknown as ProbeStore;
|
||||
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget(targetId)]);
|
||||
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||
|
||||
const stateChangeLogs = logger.entries.filter(
|
||||
(e) => e.msg.includes("UP → DOWN") || e.msg.includes("DOWN → UP") || e.msg.includes("首次 DOWN"),
|
||||
@@ -477,25 +466,17 @@ describe("ProbeEngine", () => {
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = async (target) => {
|
||||
if (target.id === targetId) {
|
||||
return {
|
||||
detail: null,
|
||||
return makeMockResult(targetId, {
|
||||
durationMs: 10,
|
||||
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
||||
matched: false,
|
||||
observation: null,
|
||||
targetId,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
}
|
||||
return originalExecute(target, { signal: new AbortController().signal });
|
||||
};
|
||||
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget(targetId)]);
|
||||
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||
|
||||
const firstDownLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("首次 DOWN"));
|
||||
expect(firstDownLogs.length).toBe(1);
|
||||
@@ -510,11 +491,7 @@ describe("ProbeEngine", () => {
|
||||
const mockStore = createMockStore([targetId]) as unknown as ProbeStore;
|
||||
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget(targetId)]);
|
||||
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||
|
||||
const debugLogs = logger.entries.filter((e) => e.level === "debug");
|
||||
expect(debugLogs.length).toBeGreaterThanOrEqual(1);
|
||||
@@ -527,11 +504,186 @@ describe("ProbeEngine", () => {
|
||||
ensureRegistered();
|
||||
const mockStore = createMockStore(["no-log"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget("no-log")]);
|
||||
await getRunOnce(engine)(makeCommandTarget("no-log"));
|
||||
});
|
||||
});
|
||||
|
||||
const probeGroup = (
|
||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
||||
).probeGroup.bind(engine);
|
||||
await probeGroup([makeCommandTarget("no-log")]);
|
||||
describe("runLoop 调度行为", () => {
|
||||
test("首次立即执行", async () => {
|
||||
ensureRegistered();
|
||||
let callCount = 0;
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = (target) => {
|
||||
callCount++;
|
||||
return Promise.resolve(makeMockResult(target.id));
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("immediate", { intervalMs: 60000 });
|
||||
const mockStore = createMockStore(["immediate"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(20);
|
||||
expect(callCount).toBeGreaterThanOrEqual(1);
|
||||
engine.stop();
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("正常调度间隔", async () => {
|
||||
ensureRegistered();
|
||||
const callTimes: number[] = [];
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = (target) => {
|
||||
callTimes.push(performance.now());
|
||||
return Promise.resolve(makeMockResult(target.id));
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("interval", { intervalMs: 100 });
|
||||
const mockStore = createMockStore(["interval"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(280);
|
||||
engine.stop();
|
||||
|
||||
expect(callTimes.length).toBeGreaterThanOrEqual(2);
|
||||
const gap = callTimes[1]! - callTimes[0]!;
|
||||
expect(gap).toBeGreaterThanOrEqual(80);
|
||||
expect(gap).toBeLessThan(200);
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("catch-up 语义:超时后立即补执行", async () => {
|
||||
ensureRegistered();
|
||||
const callTimes: number[] = [];
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = async (target) => {
|
||||
callTimes.push(performance.now());
|
||||
if (callTimes.length === 1) {
|
||||
await Bun.sleep(150);
|
||||
}
|
||||
return makeMockResult(target.id);
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("catchup", { intervalMs: 100 });
|
||||
const mockStore = createMockStore(["catchup"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(350);
|
||||
engine.stop();
|
||||
|
||||
expect(callTimes.length).toBeGreaterThanOrEqual(2);
|
||||
const gap = callTimes[1]! - callTimes[0]!;
|
||||
expect(gap).toBeGreaterThanOrEqual(140);
|
||||
expect(gap).toBeLessThan(220);
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("overrun warn 日志", async () => {
|
||||
ensureRegistered();
|
||||
const logger = createMemoryLogger();
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = async (target) => {
|
||||
await Bun.sleep(150);
|
||||
return makeMockResult(target.id, { durationMs: 150 });
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("overrun", { intervalMs: 100 });
|
||||
const mockStore = createMockStore(["overrun"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target], 20, 0, logger);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(250);
|
||||
engine.stop();
|
||||
|
||||
const warnLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("拨测超时"));
|
||||
expect(warnLogs.length).toBeGreaterThanOrEqual(1);
|
||||
expect(warnLogs[0]!.obj).toBeDefined();
|
||||
expect(warnLogs[0]!.obj!["targetId"]).toBe("overrun");
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("无并发重叠:同一 target 不会并发执行", async () => {
|
||||
ensureRegistered();
|
||||
let running = 0;
|
||||
let maxConcurrent = 0;
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = async (target) => {
|
||||
running++;
|
||||
maxConcurrent = Math.max(maxConcurrent, running);
|
||||
await Bun.sleep(60);
|
||||
running--;
|
||||
return makeMockResult(target.id, { durationMs: 60 });
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("no-overlap", { intervalMs: 70 });
|
||||
const mockStore = createMockStore(["no-overlap"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(350);
|
||||
engine.stop();
|
||||
|
||||
expect(maxConcurrent).toBeLessThanOrEqual(1);
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("优雅停止:stop() 后循环快速退出", async () => {
|
||||
ensureRegistered();
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = (target) => Promise.resolve(makeMockResult(target.id));
|
||||
|
||||
const target = makeCommandTarget("graceful", { intervalMs: 60000 });
|
||||
const mockStore = createMockStore(["graceful"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(20);
|
||||
const stopStart = performance.now();
|
||||
engine.stop();
|
||||
const stopDuration = performance.now() - stopStart;
|
||||
|
||||
expect(stopDuration).toBeLessThan(1000);
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
|
||||
test("错误隔离:runCheck 抛异常后循环继续", async () => {
|
||||
ensureRegistered();
|
||||
let callCount = 0;
|
||||
const checker = checkerRegistry.get("cmd");
|
||||
const originalExecute = checker.execute.bind(checker);
|
||||
checker.execute = (target) => {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
return Promise.reject(new Error("first fail"));
|
||||
}
|
||||
return Promise.resolve(makeMockResult(target.id));
|
||||
};
|
||||
|
||||
const target = makeCommandTarget("error-isolation", { intervalMs: 50 });
|
||||
const mockStore = createMockStore(["error-isolation"]) as unknown as ProbeStore;
|
||||
const engine = new ProbeEngine(mockStore, [target]);
|
||||
engine.start();
|
||||
|
||||
await Bun.sleep(180);
|
||||
engine.stop();
|
||||
|
||||
expect(callCount).toBeGreaterThanOrEqual(2);
|
||||
|
||||
checker.execute = originalExecute;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user