refactor: ProbeEngine 调度引擎重写为 per-target setTimeout 链
将 per-group setInterval + groupBy 调度模式改为 per-target setTimeout 链, 实现 catch-up 语义(超时后立即补执行)、AbortController 优雅停止、 循环内错误隔离和 overrun warn 日志。 移除 groupBy/probeGroup/timers,新增 sleep/runLoop/runOnce。 新增 croner 依赖供后续 cron 表达式支持使用。
This commit is contained in:
3
bun.lock
3
bun.lock
@@ -14,6 +14,7 @@
|
|||||||
"ai": "^6",
|
"ai": "^6",
|
||||||
"ajv": "^8.20.0",
|
"ajv": "^8.20.0",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
|
"croner": "^10.0.1",
|
||||||
"es-toolkit": "^1.46.1",
|
"es-toolkit": "^1.46.1",
|
||||||
"pino": "^10.3.1",
|
"pino": "^10.3.1",
|
||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
@@ -497,6 +498,8 @@
|
|||||||
|
|
||||||
"cosmiconfig-typescript-loader": ["cosmiconfig-typescript-loader@6.3.0", "https://registry.npmmirror.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.3.0.tgz", { "dependencies": { "jiti": "2.6.1" }, "peerDependencies": { "@types/node": "*", "cosmiconfig": ">=9", "typescript": ">=5" } }, "sha512-Akr82WH1Wfqatyiqpj8HDkO2o2KmJRu1FhKfSNJP3K4IdXwHfEyL7MOb62i1AGQVLtIQM+iCE9CGOtrfhR+mmA=="],
|
"cosmiconfig-typescript-loader": ["cosmiconfig-typescript-loader@6.3.0", "https://registry.npmmirror.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.3.0.tgz", { "dependencies": { "jiti": "2.6.1" }, "peerDependencies": { "@types/node": "*", "cosmiconfig": ">=9", "typescript": ">=5" } }, "sha512-Akr82WH1Wfqatyiqpj8HDkO2o2KmJRu1FhKfSNJP3K4IdXwHfEyL7MOb62i1AGQVLtIQM+iCE9CGOtrfhR+mmA=="],
|
||||||
|
|
||||||
|
"croner": ["croner@10.0.1", "", {}, "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g=="],
|
||||||
|
|
||||||
"cross-spawn": ["cross-spawn@7.0.6", "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.6.tgz", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
"cross-spawn": ["cross-spawn@7.0.6", "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.6.tgz", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||||
|
|
||||||
"css-select": ["css-select@5.2.2", "https://registry.npmmirror.com/css-select/-/css-select-5.2.2.tgz", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
"css-select": ["css-select@5.2.2", "https://registry.npmmirror.com/css-select/-/css-select-5.2.2.tgz", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||||
|
|||||||
@@ -63,6 +63,7 @@
|
|||||||
"ai": "^6",
|
"ai": "^6",
|
||||||
"ajv": "^8.20.0",
|
"ajv": "^8.20.0",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
|
"croner": "^10.0.1",
|
||||||
"es-toolkit": "^1.46.1",
|
"es-toolkit": "^1.46.1",
|
||||||
"pino": "^10.3.1",
|
"pino": "^10.3.1",
|
||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { groupBy, isError, Semaphore } from "es-toolkit";
|
import { isError, Semaphore } from "es-toolkit";
|
||||||
|
|
||||||
import type { Logger } from "../logger";
|
import type { Logger } from "../logger";
|
||||||
import type { ProbeStore } from "./store";
|
import type { ProbeStore } from "./store";
|
||||||
@@ -11,14 +11,15 @@ import { checkerRegistry } from "./runner";
|
|||||||
const PRUNE_INTERVAL_MS = 3600000;
|
const PRUNE_INTERVAL_MS = 3600000;
|
||||||
|
|
||||||
export class ProbeEngine {
|
export class ProbeEngine {
|
||||||
|
private abort: AbortController | null = null;
|
||||||
private lastMatched = new Map<string, boolean>();
|
private lastMatched = new Map<string, boolean>();
|
||||||
private logger: Logger;
|
private logger: Logger;
|
||||||
|
private pruneTimer: null | ReturnType<typeof setInterval> = null;
|
||||||
private retentionMs: number;
|
private retentionMs: number;
|
||||||
private semaphore: Semaphore;
|
private semaphore: Semaphore;
|
||||||
private store: ProbeStore;
|
private store: ProbeStore;
|
||||||
private targetIds = new Set<string>();
|
private targetIds = new Set<string>();
|
||||||
private targets: ResolvedTargetBase[];
|
private targets: ResolvedTargetBase[];
|
||||||
private timers: Array<ReturnType<typeof setInterval>> = [];
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
store: ProbeStore,
|
store: ProbeStore,
|
||||||
@@ -37,32 +38,28 @@ export class ProbeEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
start(): void {
|
start(): void {
|
||||||
const groups = groupBy(this.targets, (t) => t.intervalMs);
|
this.abort = new AbortController();
|
||||||
|
const signal = this.abort.signal;
|
||||||
|
|
||||||
for (const [intervalMs, groupTargets] of Object.entries(groups)) {
|
for (const target of this.targets) {
|
||||||
void this.probeGroup(groupTargets);
|
void this.runLoop(target, signal);
|
||||||
|
|
||||||
const timer = setInterval(() => {
|
|
||||||
void this.probeGroup(groupTargets);
|
|
||||||
}, Number(intervalMs));
|
|
||||||
|
|
||||||
this.timers.push(timer);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.retentionMs > 0) {
|
if (this.retentionMs > 0) {
|
||||||
this.store.prune(this.retentionMs);
|
this.store.prune(this.retentionMs);
|
||||||
const pruneTimer = setInterval(() => {
|
this.pruneTimer = setInterval(() => {
|
||||||
this.store.prune(this.retentionMs);
|
this.store.prune(this.retentionMs);
|
||||||
}, PRUNE_INTERVAL_MS);
|
}, PRUNE_INTERVAL_MS);
|
||||||
this.timers.push(pruneTimer);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
stop(): void {
|
stop(): void {
|
||||||
for (const timer of this.timers) {
|
this.abort?.abort();
|
||||||
clearInterval(timer);
|
this.abort = null;
|
||||||
|
if (this.pruneTimer) {
|
||||||
|
clearInterval(this.pruneTimer);
|
||||||
|
this.pruneTimer = null;
|
||||||
}
|
}
|
||||||
this.timers = [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private initStateCache(): void {
|
private initStateCache(): void {
|
||||||
@@ -108,44 +105,6 @@ export class ProbeEngine {
|
|||||||
this.lastMatched.set(result.targetId, current);
|
this.lastMatched.set(result.targetId, current);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async probeGroup(targets: ResolvedTargetBase[]): Promise<void> {
|
|
||||||
const results = await Promise.allSettled(
|
|
||||||
targets.map(async (target) => {
|
|
||||||
await this.semaphore.acquire();
|
|
||||||
try {
|
|
||||||
return await this.runCheck(target);
|
|
||||||
} finally {
|
|
||||||
this.semaphore.release();
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const [index, result] of results.entries()) {
|
|
||||||
if (result.status === "fulfilled") {
|
|
||||||
this.writeResult(result.value);
|
|
||||||
this.logStateChange(result.value);
|
|
||||||
this.logCheckDebug(result.value);
|
|
||||||
} else {
|
|
||||||
const target = targets[index];
|
|
||||||
if (target) {
|
|
||||||
this.logger.error(
|
|
||||||
{ reason: formatReason(result.reason), targetId: target.id, targetType: target.type },
|
|
||||||
`探针执行失败: ${formatReason(result.reason)}`,
|
|
||||||
);
|
|
||||||
this.writeResult({
|
|
||||||
detail: null,
|
|
||||||
durationMs: null,
|
|
||||||
failure: errorFailure("internal", "engine", formatReason(result.reason)),
|
|
||||||
matched: false,
|
|
||||||
observation: null,
|
|
||||||
targetId: target.id,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private refreshCache(): void {
|
private refreshCache(): void {
|
||||||
this.targetIds.clear();
|
this.targetIds.clear();
|
||||||
for (const target of this.store.getTargets()) {
|
for (const target of this.store.getTargets()) {
|
||||||
@@ -165,6 +124,62 @@ export class ProbeEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async runLoop(target: ResolvedTargetBase, signal: AbortSignal): Promise<void> {
|
||||||
|
while (!signal.aborted) {
|
||||||
|
const start = performance.now();
|
||||||
|
try {
|
||||||
|
await this.runOnce(target, signal);
|
||||||
|
} catch {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const elapsed = performance.now() - start;
|
||||||
|
if (elapsed > target.intervalMs) {
|
||||||
|
this.logger.warn(
|
||||||
|
{ elapsed, intervalMs: target.intervalMs, targetId: target.id },
|
||||||
|
`拨测超时: ${target.id} 耗时 ${Math.round(elapsed)}ms > 间隔 ${target.intervalMs}ms`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const delay = Math.max(0, target.intervalMs - elapsed);
|
||||||
|
try {
|
||||||
|
await sleep(delay, signal);
|
||||||
|
} catch {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runOnce(target: ResolvedTargetBase, signal?: AbortSignal): Promise<CheckResult> {
|
||||||
|
await this.semaphore.acquire();
|
||||||
|
if (signal?.aborted) {
|
||||||
|
this.semaphore.release();
|
||||||
|
throw new DOMException("Aborted", "AbortError");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const result = await this.runCheck(target);
|
||||||
|
this.writeResult(result);
|
||||||
|
this.logStateChange(result);
|
||||||
|
this.logCheckDebug(result);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
const reason = formatReason(error);
|
||||||
|
this.logger.error({ reason, targetId: target.id, targetType: target.type }, `探针执行失败: ${reason}`);
|
||||||
|
const errorResult: CheckResult = {
|
||||||
|
detail: null,
|
||||||
|
durationMs: null,
|
||||||
|
failure: errorFailure("internal", "engine", reason),
|
||||||
|
matched: false,
|
||||||
|
observation: null,
|
||||||
|
targetId: target.id,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
this.writeResult(errorResult);
|
||||||
|
return errorResult;
|
||||||
|
} finally {
|
||||||
|
this.semaphore.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private writeResult(result: CheckResult): void {
|
private writeResult(result: CheckResult): void {
|
||||||
if (!this.targetIds.has(result.targetId)) return;
|
if (!this.targetIds.has(result.targetId)) return;
|
||||||
|
|
||||||
@@ -182,3 +197,24 @@ export class ProbeEngine {
|
|||||||
function formatReason(reason: unknown): string {
|
function formatReason(reason: unknown): string {
|
||||||
return isError(reason) ? reason.message : String(reason);
|
return isError(reason) ? reason.message : String(reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
if (signal.aborted) {
|
||||||
|
reject(new DOMException("Aborted", "AbortError"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
|
||||||
|
function onAbort() {
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(new DOMException("Aborted", "AbortError"));
|
||||||
|
}
|
||||||
|
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -50,6 +50,14 @@ function ensureRegistered() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getRunOnce(engine: ProbeEngine) {
|
||||||
|
return (
|
||||||
|
engine as unknown as {
|
||||||
|
runOnce: (t: ResolvedTargetBase) => Promise<Record<string, unknown>>;
|
||||||
|
}
|
||||||
|
).runOnce.bind(engine);
|
||||||
|
}
|
||||||
|
|
||||||
function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarget>): ResolvedCommandTarget {
|
function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarget>): ResolvedCommandTarget {
|
||||||
return {
|
return {
|
||||||
cmd: {
|
cmd: {
|
||||||
@@ -70,6 +78,19 @@ function makeCommandTarget(name: string, overrides?: Partial<ResolvedCommandTarg
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function makeMockResult(targetId: string, overrides?: Partial<Record<string, unknown>>) {
|
||||||
|
return {
|
||||||
|
detail: null,
|
||||||
|
durationMs: 1,
|
||||||
|
failure: null,
|
||||||
|
matched: true,
|
||||||
|
observation: null,
|
||||||
|
targetId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe("ProbeEngine", () => {
|
describe("ProbeEngine", () => {
|
||||||
test("start/stop 不抛错", () => {
|
test("start/stop 不抛错", () => {
|
||||||
ensureRegistered();
|
ensureRegistered();
|
||||||
@@ -81,15 +102,12 @@ describe("ProbeEngine", () => {
|
|||||||
expect(true).toBe(true);
|
expect(true).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("单次 probeGroup 执行 cmd 检查", async () => {
|
test("单次 runOnce 执行 cmd 检查", async () => {
|
||||||
const target = makeCommandTarget("cmd-echo");
|
const target = makeCommandTarget("cmd-echo");
|
||||||
const mockStore = createMockStore(["cmd-echo"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["cmd-echo"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [target]);
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
|
||||||
const probeGroup = (
|
await getRunOnce(engine)(target);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([target]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
@@ -119,11 +137,9 @@ describe("ProbeEngine", () => {
|
|||||||
|
|
||||||
const mockStore = createMockStore(["echo-a", "echo-b"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["echo-a", "echo-b"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [targetA, targetB]);
|
const engine = new ProbeEngine(mockStore, [targetA, targetB]);
|
||||||
|
const runOnce = getRunOnce(engine);
|
||||||
|
|
||||||
const probeGroup = (
|
await Promise.all([runOnce(targetA), runOnce(targetB)]);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([targetA, targetB]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(2);
|
expect(results.length).toBe(2);
|
||||||
@@ -143,11 +159,9 @@ describe("ProbeEngine", () => {
|
|||||||
|
|
||||||
const mockStore = createMockStore(["bad-cmd", "good-cmd"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["bad-cmd", "good-cmd"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [badTarget, goodTarget]);
|
const engine = new ProbeEngine(mockStore, [badTarget, goodTarget]);
|
||||||
|
const runOnce = getRunOnce(engine);
|
||||||
|
|
||||||
const probeGroup = (
|
await Promise.all([runOnce(badTarget), runOnce(goodTarget)]);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([badTarget, goodTarget]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(2);
|
expect(results.length).toBe(2);
|
||||||
@@ -173,27 +187,28 @@ describe("ProbeEngine", () => {
|
|||||||
const goodTarget = makeCommandTarget("good-cmd");
|
const goodTarget = makeCommandTarget("good-cmd");
|
||||||
const mockStore = createMockStore(["reject-cmd", "good-cmd"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["reject-cmd", "good-cmd"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [rejectTarget, goodTarget]);
|
const engine = new ProbeEngine(mockStore, [rejectTarget, goodTarget]);
|
||||||
|
const runOnce = getRunOnce(engine);
|
||||||
|
|
||||||
const probeGroup = (
|
await Promise.all([runOnce(rejectTarget), runOnce(goodTarget)]);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([rejectTarget, goodTarget]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(2);
|
expect(results.length).toBe(2);
|
||||||
expect(results[0]!["targetId"]).toBe("reject-cmd");
|
|
||||||
expect(results[0]!["matched"]).toBe(false);
|
const rejectResult = results.find((r) => r["targetId"] === "reject-cmd");
|
||||||
expect(results[0]!["durationMs"]).toBeNull();
|
const goodResult = results.find((r) => r["targetId"] === "good-cmd");
|
||||||
expect(results[0]!["observation"]).toBeNull();
|
expect(rejectResult).toBeDefined();
|
||||||
expect(results[0]!["failure"]).toEqual({
|
expect(rejectResult!["matched"]).toBe(false);
|
||||||
|
expect(rejectResult!["durationMs"]).toBeNull();
|
||||||
|
expect(rejectResult!["observation"]).toBeNull();
|
||||||
|
expect(rejectResult!["failure"]).toEqual({
|
||||||
kind: "error",
|
kind: "error",
|
||||||
message: "boom",
|
message: "boom",
|
||||||
path: "engine",
|
path: "engine",
|
||||||
phase: "internal",
|
phase: "internal",
|
||||||
});
|
});
|
||||||
expect(typeof results[0]!["timestamp"]).toBe("string");
|
expect(typeof rejectResult!["timestamp"]).toBe("string");
|
||||||
expect(results[1]!["targetId"]).toBe("good-cmd");
|
expect(goodResult).toBeDefined();
|
||||||
expect(results[1]!["matched"]).toBe(true);
|
expect(goodResult!["matched"]).toBe(true);
|
||||||
checker.execute = originalExecute;
|
checker.execute = originalExecute;
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -212,11 +227,9 @@ describe("ProbeEngine", () => {
|
|||||||
|
|
||||||
const mockStore = createMockStore(targets.map((t) => t.name ?? t.id)) as unknown as ProbeStore;
|
const mockStore = createMockStore(targets.map((t) => t.name ?? t.id)) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, targets, 2);
|
const engine = new ProbeEngine(mockStore, targets, 2);
|
||||||
|
const runOnce = getRunOnce(engine);
|
||||||
|
|
||||||
const probeGroup = (
|
await Promise.all(targets.map((t) => runOnce(t)));
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup(targets);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(5);
|
expect(results.length).toBe(5);
|
||||||
@@ -225,7 +238,7 @@ describe("ProbeEngine", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("groupByInterval 按间隔分组", () => {
|
test("不同 interval 的 target 独立调度", () => {
|
||||||
const targetA = makeCommandTarget("a", { intervalMs: 30000 });
|
const targetA = makeCommandTarget("a", { intervalMs: 30000 });
|
||||||
const targetB = makeCommandTarget("b", { intervalMs: 30000 });
|
const targetB = makeCommandTarget("b", { intervalMs: 30000 });
|
||||||
const targetC = makeCommandTarget("c", { intervalMs: 60000 });
|
const targetC = makeCommandTarget("c", { intervalMs: 60000 });
|
||||||
@@ -242,10 +255,7 @@ describe("ProbeEngine", () => {
|
|||||||
const mockStore = createMockStore(["other-name"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["other-name"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [target]);
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
|
||||||
const probeGroup = (
|
await getRunOnce(engine)(target);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([target]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(0);
|
expect(results.length).toBe(0);
|
||||||
@@ -281,10 +291,7 @@ describe("ProbeEngine", () => {
|
|||||||
const mockStore = createMockStore(["http-test"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["http-test"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [httpTarget]);
|
const engine = new ProbeEngine(mockStore, [httpTarget]);
|
||||||
|
|
||||||
const probeGroup = (
|
await getRunOnce(engine)(httpTarget);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([httpTarget]);
|
|
||||||
|
|
||||||
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
const results = (mockStore as unknown as { _results: Array<Record<string, unknown>> })._results;
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
@@ -365,11 +372,9 @@ describe("ProbeEngine", () => {
|
|||||||
0,
|
0,
|
||||||
logger,
|
logger,
|
||||||
);
|
);
|
||||||
|
const runOnce = getRunOnce(engine);
|
||||||
|
|
||||||
const probeGroup = (
|
await Promise.all([runOnce(makeCommandTarget("fail-target")), runOnce(makeCommandTarget("ok-target"))]);
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget("fail-target"), makeCommandTarget("ok-target")]);
|
|
||||||
|
|
||||||
const errorLogs = logger.entries.filter((e) => e.level === "error");
|
const errorLogs = logger.entries.filter((e) => e.level === "error");
|
||||||
expect(errorLogs.length).toBeGreaterThanOrEqual(1);
|
expect(errorLogs.length).toBeGreaterThanOrEqual(1);
|
||||||
@@ -393,25 +398,17 @@ describe("ProbeEngine", () => {
|
|||||||
const originalExecute = checker.execute.bind(checker);
|
const originalExecute = checker.execute.bind(checker);
|
||||||
checker.execute = async (target) => {
|
checker.execute = async (target) => {
|
||||||
if (target.id === targetId) {
|
if (target.id === targetId) {
|
||||||
return {
|
return makeMockResult(targetId, {
|
||||||
detail: null,
|
|
||||||
durationMs: 10,
|
durationMs: 10,
|
||||||
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
||||||
matched: false,
|
matched: false,
|
||||||
observation: null,
|
});
|
||||||
targetId,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
return originalExecute(target, { signal: new AbortController().signal });
|
return originalExecute(target, { signal: new AbortController().signal });
|
||||||
};
|
};
|
||||||
|
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||||
const probeGroup = (
|
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget(targetId)]);
|
|
||||||
|
|
||||||
const stateLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("UP → DOWN"));
|
const stateLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("UP → DOWN"));
|
||||||
expect(stateLogs.length).toBe(1);
|
expect(stateLogs.length).toBe(1);
|
||||||
@@ -432,11 +429,7 @@ describe("ProbeEngine", () => {
|
|||||||
} as unknown as ProbeStore;
|
} as unknown as ProbeStore;
|
||||||
|
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||||
const probeGroup = (
|
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget(targetId)]);
|
|
||||||
|
|
||||||
const recoverLogs = logger.entries.filter((e) => e.level === "info" && e.msg.includes("DOWN → UP"));
|
const recoverLogs = logger.entries.filter((e) => e.level === "info" && e.msg.includes("DOWN → UP"));
|
||||||
expect(recoverLogs.length).toBe(1);
|
expect(recoverLogs.length).toBe(1);
|
||||||
@@ -455,11 +448,7 @@ describe("ProbeEngine", () => {
|
|||||||
} as unknown as ProbeStore;
|
} as unknown as ProbeStore;
|
||||||
|
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||||
const probeGroup = (
|
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget(targetId)]);
|
|
||||||
|
|
||||||
const stateChangeLogs = logger.entries.filter(
|
const stateChangeLogs = logger.entries.filter(
|
||||||
(e) => e.msg.includes("UP → DOWN") || e.msg.includes("DOWN → UP") || e.msg.includes("首次 DOWN"),
|
(e) => e.msg.includes("UP → DOWN") || e.msg.includes("DOWN → UP") || e.msg.includes("首次 DOWN"),
|
||||||
@@ -477,25 +466,17 @@ describe("ProbeEngine", () => {
|
|||||||
const originalExecute = checker.execute.bind(checker);
|
const originalExecute = checker.execute.bind(checker);
|
||||||
checker.execute = async (target) => {
|
checker.execute = async (target) => {
|
||||||
if (target.id === targetId) {
|
if (target.id === targetId) {
|
||||||
return {
|
return makeMockResult(targetId, {
|
||||||
detail: null,
|
|
||||||
durationMs: 10,
|
durationMs: 10,
|
||||||
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
failure: { kind: "error", message: "fail", path: "exitCode", phase: "body" },
|
||||||
matched: false,
|
matched: false,
|
||||||
observation: null,
|
});
|
||||||
targetId,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
return originalExecute(target, { signal: new AbortController().signal });
|
return originalExecute(target, { signal: new AbortController().signal });
|
||||||
};
|
};
|
||||||
|
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||||
const probeGroup = (
|
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget(targetId)]);
|
|
||||||
|
|
||||||
const firstDownLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("首次 DOWN"));
|
const firstDownLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("首次 DOWN"));
|
||||||
expect(firstDownLogs.length).toBe(1);
|
expect(firstDownLogs.length).toBe(1);
|
||||||
@@ -510,11 +491,7 @@ describe("ProbeEngine", () => {
|
|||||||
const mockStore = createMockStore([targetId]) as unknown as ProbeStore;
|
const mockStore = createMockStore([targetId]) as unknown as ProbeStore;
|
||||||
|
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget(targetId)], 20, 0, logger);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget(targetId));
|
||||||
const probeGroup = (
|
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
|
||||||
).probeGroup.bind(engine);
|
|
||||||
await probeGroup([makeCommandTarget(targetId)]);
|
|
||||||
|
|
||||||
const debugLogs = logger.entries.filter((e) => e.level === "debug");
|
const debugLogs = logger.entries.filter((e) => e.level === "debug");
|
||||||
expect(debugLogs.length).toBeGreaterThanOrEqual(1);
|
expect(debugLogs.length).toBeGreaterThanOrEqual(1);
|
||||||
@@ -527,11 +504,186 @@ describe("ProbeEngine", () => {
|
|||||||
ensureRegistered();
|
ensureRegistered();
|
||||||
const mockStore = createMockStore(["no-log"]) as unknown as ProbeStore;
|
const mockStore = createMockStore(["no-log"]) as unknown as ProbeStore;
|
||||||
const engine = new ProbeEngine(mockStore, [makeCommandTarget("no-log")]);
|
const engine = new ProbeEngine(mockStore, [makeCommandTarget("no-log")]);
|
||||||
|
await getRunOnce(engine)(makeCommandTarget("no-log"));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
const probeGroup = (
|
describe("runLoop 调度行为", () => {
|
||||||
engine as unknown as { probeGroup: (t: ResolvedTargetBase[]) => Promise<void> }
|
test("首次立即执行", async () => {
|
||||||
).probeGroup.bind(engine);
|
ensureRegistered();
|
||||||
await probeGroup([makeCommandTarget("no-log")]);
|
let callCount = 0;
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = (target) => {
|
||||||
|
callCount++;
|
||||||
|
return Promise.resolve(makeMockResult(target.id));
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("immediate", { intervalMs: 60000 });
|
||||||
|
const mockStore = createMockStore(["immediate"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(20);
|
||||||
|
expect(callCount).toBeGreaterThanOrEqual(1);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("正常调度间隔", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
const callTimes: number[] = [];
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = (target) => {
|
||||||
|
callTimes.push(performance.now());
|
||||||
|
return Promise.resolve(makeMockResult(target.id));
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("interval", { intervalMs: 100 });
|
||||||
|
const mockStore = createMockStore(["interval"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(280);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
expect(callTimes.length).toBeGreaterThanOrEqual(2);
|
||||||
|
const gap = callTimes[1]! - callTimes[0]!;
|
||||||
|
expect(gap).toBeGreaterThanOrEqual(80);
|
||||||
|
expect(gap).toBeLessThan(200);
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("catch-up 语义:超时后立即补执行", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
const callTimes: number[] = [];
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = async (target) => {
|
||||||
|
callTimes.push(performance.now());
|
||||||
|
if (callTimes.length === 1) {
|
||||||
|
await Bun.sleep(150);
|
||||||
|
}
|
||||||
|
return makeMockResult(target.id);
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("catchup", { intervalMs: 100 });
|
||||||
|
const mockStore = createMockStore(["catchup"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(350);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
expect(callTimes.length).toBeGreaterThanOrEqual(2);
|
||||||
|
const gap = callTimes[1]! - callTimes[0]!;
|
||||||
|
expect(gap).toBeGreaterThanOrEqual(140);
|
||||||
|
expect(gap).toBeLessThan(220);
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("overrun warn 日志", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
const logger = createMemoryLogger();
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = async (target) => {
|
||||||
|
await Bun.sleep(150);
|
||||||
|
return makeMockResult(target.id, { durationMs: 150 });
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("overrun", { intervalMs: 100 });
|
||||||
|
const mockStore = createMockStore(["overrun"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target], 20, 0, logger);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(250);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
const warnLogs = logger.entries.filter((e) => e.level === "warn" && e.msg.includes("拨测超时"));
|
||||||
|
expect(warnLogs.length).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(warnLogs[0]!.obj).toBeDefined();
|
||||||
|
expect(warnLogs[0]!.obj!["targetId"]).toBe("overrun");
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("无并发重叠:同一 target 不会并发执行", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
let running = 0;
|
||||||
|
let maxConcurrent = 0;
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = async (target) => {
|
||||||
|
running++;
|
||||||
|
maxConcurrent = Math.max(maxConcurrent, running);
|
||||||
|
await Bun.sleep(60);
|
||||||
|
running--;
|
||||||
|
return makeMockResult(target.id, { durationMs: 60 });
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("no-overlap", { intervalMs: 70 });
|
||||||
|
const mockStore = createMockStore(["no-overlap"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(350);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
expect(maxConcurrent).toBeLessThanOrEqual(1);
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("优雅停止:stop() 后循环快速退出", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = (target) => Promise.resolve(makeMockResult(target.id));
|
||||||
|
|
||||||
|
const target = makeCommandTarget("graceful", { intervalMs: 60000 });
|
||||||
|
const mockStore = createMockStore(["graceful"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(20);
|
||||||
|
const stopStart = performance.now();
|
||||||
|
engine.stop();
|
||||||
|
const stopDuration = performance.now() - stopStart;
|
||||||
|
|
||||||
|
expect(stopDuration).toBeLessThan(1000);
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("错误隔离:runCheck 抛异常后循环继续", async () => {
|
||||||
|
ensureRegistered();
|
||||||
|
let callCount = 0;
|
||||||
|
const checker = checkerRegistry.get("cmd");
|
||||||
|
const originalExecute = checker.execute.bind(checker);
|
||||||
|
checker.execute = (target) => {
|
||||||
|
callCount++;
|
||||||
|
if (callCount === 1) {
|
||||||
|
return Promise.reject(new Error("first fail"));
|
||||||
|
}
|
||||||
|
return Promise.resolve(makeMockResult(target.id));
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = makeCommandTarget("error-isolation", { intervalMs: 50 });
|
||||||
|
const mockStore = createMockStore(["error-isolation"]) as unknown as ProbeStore;
|
||||||
|
const engine = new ProbeEngine(mockStore, [target]);
|
||||||
|
engine.start();
|
||||||
|
|
||||||
|
await Bun.sleep(180);
|
||||||
|
engine.stop();
|
||||||
|
|
||||||
|
expect(callCount).toBeGreaterThanOrEqual(2);
|
||||||
|
|
||||||
|
checker.execute = originalExecute;
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user