docs: enhance heartbeat service descriptions and CLI registration
This commit is contained in:
10
index.ts
10
index.ts
@@ -63,22 +63,22 @@ const plugin = {
|
|||||||
work_heartbeat: {
|
work_heartbeat: {
|
||||||
type: "object",
|
type: "object",
|
||||||
description:
|
description:
|
||||||
"Token-free interval-based heartbeat service. Runs health checks + queue dispatch automatically.",
|
"Token-free interval-based heartbeat service. Runs health checks + queue dispatch automatically. Discovers all DevClaw agents from openclaw.json and processes each independently. Can also be triggered on-demand via CLI command `devclaw heartbeat:tick`.",
|
||||||
properties: {
|
properties: {
|
||||||
enabled: {
|
enabled: {
|
||||||
type: "boolean",
|
type: "boolean",
|
||||||
default: true,
|
default: true,
|
||||||
description: "Enable the heartbeat service.",
|
description: "Enable automatic periodic heartbeat service. When disabled, heartbeat can still be run on-demand via `devclaw heartbeat:tick` CLI command.",
|
||||||
},
|
},
|
||||||
intervalSeconds: {
|
intervalSeconds: {
|
||||||
type: "number",
|
type: "number",
|
||||||
default: 60,
|
default: 60,
|
||||||
description: "Seconds between ticks.",
|
description: "Seconds between automatic heartbeat ticks (only applies when service is enabled). Can be overridden per-tick via CLI option.",
|
||||||
},
|
},
|
||||||
maxPickupsPerTick: {
|
maxPickupsPerTick: {
|
||||||
type: "number",
|
type: "number",
|
||||||
default: 4,
|
default: 4,
|
||||||
description: "Max worker dispatches per tick.",
|
description: "Max worker dispatches per agent per tick. Applied to each DevClaw agent independently.",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -118,7 +118,7 @@ const plugin = {
|
|||||||
registerHeartbeatService(api);
|
registerHeartbeatService(api);
|
||||||
|
|
||||||
api.logger.info(
|
api.logger.info(
|
||||||
"DevClaw plugin registered (11 tools, 1 service, 1 CLI command)",
|
"DevClaw plugin registered (11 tools, 1 CLI command group, 1 service)",
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* cli.ts — CLI registration for `openclaw devclaw setup`.
|
* cli.ts — CLI registration for `openclaw devclaw setup` and `openclaw devclaw heartbeat`.
|
||||||
*
|
*
|
||||||
* Uses Commander.js (provided by OpenClaw plugin SDK context).
|
* Uses Commander.js (provided by OpenClaw plugin SDK context).
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
/**
|
/**
|
||||||
* Heartbeat service — token-free interval-based queue processing.
|
* Heartbeat tick — token-free queue processing.
|
||||||
*
|
*
|
||||||
* Runs as a plugin service (tied to gateway lifecycle). Every N seconds:
|
* Runs automatically via plugin service (periodic execution).
|
||||||
|
*
|
||||||
|
* Logic:
|
||||||
* 1. Health pass: auto-fix zombies, stale workers, orphaned state
|
* 1. Health pass: auto-fix zombies, stale workers, orphaned state
|
||||||
* 2. Tick pass: fill free worker slots by priority
|
* 2. Tick pass: fill free worker slots by priority
|
||||||
*
|
*
|
||||||
@@ -9,14 +11,14 @@
|
|||||||
* Workers only consume tokens when they start processing dispatched tasks.
|
* Workers only consume tokens when they start processing dispatched tasks.
|
||||||
*/
|
*/
|
||||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
||||||
import { readProjects, getProject } from "../projects.js";
|
import { readProjects } from "../projects.js";
|
||||||
import { log as auditLog } from "../audit.js";
|
import { log as auditLog } from "../audit.js";
|
||||||
import { checkWorkerHealth } from "./health.js";
|
import { checkWorkerHealth } from "./health.js";
|
||||||
import { projectTick } from "./tick.js";
|
import { projectTick } from "./tick.js";
|
||||||
import { createProvider } from "../providers/index.js";
|
import { createProvider } from "../providers/index.js";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Config
|
// Types
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export type HeartbeatConfig = {
|
export type HeartbeatConfig = {
|
||||||
@@ -25,6 +27,31 @@ export type HeartbeatConfig = {
|
|||||||
maxPickupsPerTick: number;
|
maxPickupsPerTick: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type Agent = {
|
||||||
|
agentId: string;
|
||||||
|
workspace: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type TickResult = {
|
||||||
|
totalPickups: number;
|
||||||
|
totalHealthFixes: number;
|
||||||
|
totalSkipped: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ServiceContext = {
|
||||||
|
logger: { info(msg: string): void; warn(msg: string): void; error(msg: string): void };
|
||||||
|
config: {
|
||||||
|
agents?: { list?: Array<{ id: string; workspace?: string }> };
|
||||||
|
plugins?: {
|
||||||
|
entries?: { devclaw?: { config?: { devClawAgentIds?: string[] } } };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Config
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export const HEARTBEAT_DEFAULTS: HeartbeatConfig = {
|
export const HEARTBEAT_DEFAULTS: HeartbeatConfig = {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
intervalSeconds: 60,
|
intervalSeconds: 60,
|
||||||
@@ -48,7 +75,7 @@ export function registerHeartbeatService(api: OpenClawPluginApi) {
|
|||||||
api.registerService({
|
api.registerService({
|
||||||
id: "devclaw-heartbeat",
|
id: "devclaw-heartbeat",
|
||||||
|
|
||||||
start: async (ctx) => {
|
start: async (ctx: ServiceContext) => {
|
||||||
const pluginConfig = api.pluginConfig as Record<string, unknown> | undefined;
|
const pluginConfig = api.pluginConfig as Record<string, unknown> | undefined;
|
||||||
const config = resolveHeartbeatConfig(pluginConfig);
|
const config = resolveHeartbeatConfig(pluginConfig);
|
||||||
|
|
||||||
@@ -57,25 +84,20 @@ export function registerHeartbeatService(api: OpenClawPluginApi) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const workspaceDir = ctx.workspaceDir;
|
const agents = discoverAgents(ctx.config);
|
||||||
if (!workspaceDir) {
|
if (agents.length === 0) {
|
||||||
ctx.logger.warn("work_heartbeat: no workspaceDir — service not started");
|
ctx.logger.warn("work_heartbeat service: no DevClaw agents registered");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const agentId = resolveAgentId(pluginConfig);
|
|
||||||
|
|
||||||
ctx.logger.info(
|
ctx.logger.info(
|
||||||
`work_heartbeat service started: every ${config.intervalSeconds}s, max ${config.maxPickupsPerTick} pickups/tick`,
|
`work_heartbeat service started: every ${config.intervalSeconds}s, ${agents.length} agents, max ${config.maxPickupsPerTick} pickups/tick`,
|
||||||
);
|
);
|
||||||
|
|
||||||
intervalId = setInterval(async () => {
|
intervalId = setInterval(
|
||||||
try {
|
() => runHeartbeatTick(agents, config, pluginConfig, ctx.logger),
|
||||||
await tick({ workspaceDir, agentId, config, pluginConfig, logger: ctx.logger });
|
config.intervalSeconds * 1000,
|
||||||
} catch (err) {
|
);
|
||||||
ctx.logger.error(`work_heartbeat tick failed: ${err}`);
|
|
||||||
}
|
|
||||||
}, config.intervalSeconds * 1000);
|
|
||||||
},
|
},
|
||||||
|
|
||||||
stop: async (ctx) => {
|
stop: async (ctx) => {
|
||||||
@@ -89,92 +111,196 @@ export function registerHeartbeatService(api: OpenClawPluginApi) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Tick
|
// Helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function tick(opts: {
|
/**
|
||||||
|
* Extract DevClaw agents from OpenClaw configuration.
|
||||||
|
*/
|
||||||
|
function discoverAgents(config: ServiceContext["config"]): Agent[] {
|
||||||
|
const devClawAgentIds = config.plugins?.entries?.devclaw?.config?.devClawAgentIds || [];
|
||||||
|
const agentsList = config.agents?.list || [];
|
||||||
|
|
||||||
|
return devClawAgentIds
|
||||||
|
.map((agentId) => {
|
||||||
|
const agent = agentsList.find((a) => a.id === agentId);
|
||||||
|
return {
|
||||||
|
agentId,
|
||||||
|
workspace: agent?.workspace || "",
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((a): a is Agent => !!a.workspace);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run one heartbeat tick for all agents.
|
||||||
|
*/
|
||||||
|
async function runHeartbeatTick(
|
||||||
|
agents: Agent[],
|
||||||
|
config: HeartbeatConfig,
|
||||||
|
pluginConfig: Record<string, unknown> | undefined,
|
||||||
|
logger: ServiceContext["logger"],
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
const result = await processAllAgents(agents, config, pluginConfig, logger);
|
||||||
|
logTickResult(result, logger);
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(`work_heartbeat tick failed: ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process heartbeat tick for all agents and aggregate results.
|
||||||
|
*/
|
||||||
|
async function processAllAgents(
|
||||||
|
agents: Agent[],
|
||||||
|
config: HeartbeatConfig,
|
||||||
|
pluginConfig: Record<string, unknown> | undefined,
|
||||||
|
logger: ServiceContext["logger"],
|
||||||
|
): Promise<TickResult> {
|
||||||
|
const result: TickResult = {
|
||||||
|
totalPickups: 0,
|
||||||
|
totalHealthFixes: 0,
|
||||||
|
totalSkipped: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const { agentId, workspace } of agents) {
|
||||||
|
const agentResult = await tick({
|
||||||
|
workspaceDir: workspace,
|
||||||
|
agentId,
|
||||||
|
config,
|
||||||
|
pluginConfig,
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
|
||||||
|
result.totalPickups += agentResult.totalPickups;
|
||||||
|
result.totalHealthFixes += agentResult.totalHealthFixes;
|
||||||
|
result.totalSkipped += agentResult.totalSkipped;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log tick results if anything happened.
|
||||||
|
*/
|
||||||
|
function logTickResult(result: TickResult, logger: ServiceContext["logger"]): void {
|
||||||
|
if (result.totalPickups > 0 || result.totalHealthFixes > 0) {
|
||||||
|
logger.info(
|
||||||
|
`work_heartbeat tick: ${result.totalPickups} pickups, ${result.totalHealthFixes} health fixes, ${result.totalSkipped} skipped`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tick (Main Heartbeat Loop)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function tick(opts: {
|
||||||
workspaceDir: string;
|
workspaceDir: string;
|
||||||
agentId?: string;
|
agentId?: string;
|
||||||
config: HeartbeatConfig;
|
config: HeartbeatConfig;
|
||||||
pluginConfig?: Record<string, unknown>;
|
pluginConfig?: Record<string, unknown>;
|
||||||
logger: { info(msg: string): void; warn(msg: string): void };
|
logger: { info(msg: string): void; warn(msg: string): void };
|
||||||
}) {
|
}): Promise<TickResult> {
|
||||||
const { workspaceDir, agentId, config, pluginConfig, logger } = opts;
|
const { workspaceDir, agentId, config, pluginConfig } = opts;
|
||||||
|
|
||||||
const data = await readProjects(workspaceDir);
|
const data = await readProjects(workspaceDir);
|
||||||
const projectIds = Object.keys(data.projects);
|
const projectIds = Object.keys(data.projects);
|
||||||
if (projectIds.length === 0) return;
|
|
||||||
|
|
||||||
const projectExecution =
|
if (projectIds.length === 0) {
|
||||||
(pluginConfig?.projectExecution as string) ?? "parallel";
|
return { totalPickups: 0, totalHealthFixes: 0, totalSkipped: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
let totalPickups = 0;
|
const result: TickResult = {
|
||||||
let totalHealthFixes = 0;
|
totalPickups: 0,
|
||||||
let totalSkipped = 0;
|
totalHealthFixes: 0,
|
||||||
|
totalSkipped: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const projectExecution = (pluginConfig?.projectExecution as string) ?? "parallel";
|
||||||
let activeProjects = 0;
|
let activeProjects = 0;
|
||||||
|
|
||||||
for (const groupId of projectIds) {
|
for (const groupId of projectIds) {
|
||||||
const project = data.projects[groupId];
|
const project = data.projects[groupId];
|
||||||
if (!project) continue;
|
if (!project) continue;
|
||||||
|
|
||||||
const { provider } = createProvider({ repo: project.repo });
|
// Health pass: auto-fix zombies and stale workers
|
||||||
|
result.totalHealthFixes += await performHealthPass(
|
||||||
|
workspaceDir,
|
||||||
|
groupId,
|
||||||
|
project,
|
||||||
|
);
|
||||||
|
|
||||||
// Health pass: auto-fix
|
// Budget check: stop if we've hit the limit
|
||||||
for (const role of ["dev", "qa"] as const) {
|
const remaining = config.maxPickupsPerTick - result.totalPickups;
|
||||||
const fixes = await checkWorkerHealth({
|
|
||||||
workspaceDir, groupId, project, role,
|
|
||||||
activeSessions: [], // No session list in service context
|
|
||||||
autoFix: true,
|
|
||||||
provider,
|
|
||||||
});
|
|
||||||
totalHealthFixes += fixes.filter((f) => f.fixed).length;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Budget check
|
|
||||||
const remaining = config.maxPickupsPerTick - totalPickups;
|
|
||||||
if (remaining <= 0) break;
|
if (remaining <= 0) break;
|
||||||
|
|
||||||
// Sequential project guard
|
// Sequential project guard: don't start new projects if one is active
|
||||||
const fresh = (await readProjects(workspaceDir)).projects[groupId];
|
const isProjectActive = await checkProjectActive(workspaceDir, groupId);
|
||||||
if (!fresh) continue;
|
if (projectExecution === "sequential" && !isProjectActive && activeProjects >= 1) {
|
||||||
const projectActive = fresh.dev.active || fresh.qa.active;
|
result.totalSkipped++;
|
||||||
if (projectExecution === "sequential" && !projectActive && activeProjects >= 1) {
|
|
||||||
totalSkipped++;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tick pass: fill free slots
|
// Tick pass: fill free worker slots
|
||||||
const result = await projectTick({
|
const tickResult = await projectTick({
|
||||||
workspaceDir, groupId, agentId,
|
workspaceDir,
|
||||||
|
groupId,
|
||||||
|
agentId,
|
||||||
pluginConfig,
|
pluginConfig,
|
||||||
maxPickups: remaining,
|
maxPickups: remaining,
|
||||||
});
|
});
|
||||||
|
|
||||||
totalPickups += result.pickups.length;
|
result.totalPickups += tickResult.pickups.length;
|
||||||
totalSkipped += result.skipped.length;
|
result.totalSkipped += tickResult.skipped.length;
|
||||||
if (projectActive || result.pickups.length > 0) activeProjects++;
|
if (isProjectActive || tickResult.pickups.length > 0) activeProjects++;
|
||||||
}
|
|
||||||
|
|
||||||
// Audit (only when something happened)
|
|
||||||
if (totalPickups > 0 || totalHealthFixes > 0) {
|
|
||||||
logger.info(
|
|
||||||
`work_heartbeat tick: ${totalPickups} pickups, ${totalHealthFixes} health fixes, ${totalSkipped} skipped`,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await auditLog(workspaceDir, "heartbeat_tick", {
|
await auditLog(workspaceDir, "heartbeat_tick", {
|
||||||
projectsScanned: projectIds.length,
|
projectsScanned: projectIds.length,
|
||||||
healthFixes: totalHealthFixes,
|
healthFixes: result.totalHealthFixes,
|
||||||
pickups: totalPickups,
|
pickups: result.totalPickups,
|
||||||
skipped: totalSkipped,
|
skipped: result.totalSkipped,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
/**
|
||||||
// Helpers
|
* Run health checks and auto-fix for a project (dev + qa roles).
|
||||||
// ---------------------------------------------------------------------------
|
*/
|
||||||
|
async function performHealthPass(
|
||||||
|
workspaceDir: string,
|
||||||
|
groupId: string,
|
||||||
|
project: any,
|
||||||
|
): Promise<number> {
|
||||||
|
const { provider } = createProvider({ repo: project.repo });
|
||||||
|
let fixedCount = 0;
|
||||||
|
|
||||||
function resolveAgentId(pluginConfig?: Record<string, unknown>): string | undefined {
|
for (const role of ["dev", "qa"] as const) {
|
||||||
const ids = pluginConfig?.devClawAgentIds as string[] | undefined;
|
const fixes = await checkWorkerHealth({
|
||||||
return ids?.[0];
|
workspaceDir,
|
||||||
|
groupId,
|
||||||
|
project,
|
||||||
|
role,
|
||||||
|
activeSessions: [],
|
||||||
|
autoFix: true,
|
||||||
|
provider,
|
||||||
|
});
|
||||||
|
fixedCount += fixes.filter((f) => f.fixed).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
return fixedCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a project has active work (dev or qa).
|
||||||
|
*/
|
||||||
|
async function checkProjectActive(workspaceDir: string, groupId: string): Promise<boolean> {
|
||||||
|
const fresh = (await readProjects(workspaceDir)).projects[groupId];
|
||||||
|
if (!fresh) return false;
|
||||||
|
return fresh.dev.active || fresh.qa.active;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user