feat: enhance workflow and testing infrastructure
- Introduced ExecutionMode type for project execution modes (parallel, sequential). - Updated SetupOpts to use ExecutionMode instead of string literals. - Enhanced workflow states to include a new "In Review" state with appropriate transitions. - Implemented TestHarness for end-to-end testing, including command interception and workspace setup. - Created TestProvider for in-memory issue tracking during tests. - Refactored project registration and setup tools to utilize ExecutionMode. - Updated various tools to ensure compatibility with new workflow and execution modes. - Added new dependencies: cockatiel for resilience and zod for schema validation.
This commit is contained in:
Lauren ten Hoor
292
lib/testing/harness.ts
Normal file
292
lib/testing/harness.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Test harness — scaffolds a temporary workspace with projects.json,
|
||||
* installs a mock runCommand, and provides helpers for E2E pipeline tests.
|
||||
*
|
||||
* Usage:
|
||||
* const h = await createTestHarness({ ... });
|
||||
* try { ... } finally { await h.cleanup(); }
|
||||
*/
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { initRunCommand } from "../run-command.js";
|
||||
import { writeProjects, type ProjectsData, type Project, emptyWorkerState } from "../projects.js";
|
||||
import { DEFAULT_WORKFLOW, type WorkflowConfig } from "../workflow.js";
|
||||
import { registerBootstrapHook } from "../bootstrap-hook.js";
|
||||
import { TestProvider } from "./test-provider.js";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bootstrap file type (mirrors OpenClaw's internal type)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type BootstrapFile = {
|
||||
name: string;
|
||||
path: string;
|
||||
content?: string;
|
||||
missing: boolean;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Command interceptor
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type CapturedCommand = {
|
||||
argv: string[];
|
||||
opts: { timeoutMs: number; cwd?: string };
|
||||
/** Extracted from gateway `agent` call params, if applicable. */
|
||||
taskMessage?: string;
|
||||
/** Extracted from gateway `sessions.patch` params, if applicable. */
|
||||
sessionPatch?: { key: string; model: string };
|
||||
};
|
||||
|
||||
export type CommandInterceptor = {
|
||||
/** All captured commands, in order. */
|
||||
commands: CapturedCommand[];
|
||||
/** Filter commands by first argv element. */
|
||||
commandsFor(cmd: string): CapturedCommand[];
|
||||
/** Get all task messages sent via `openclaw gateway call agent`. */
|
||||
taskMessages(): string[];
|
||||
/** Get all session patches. */
|
||||
sessionPatches(): Array<{ key: string; model: string }>;
|
||||
/** Reset captured commands. */
|
||||
reset(): void;
|
||||
};
|
||||
|
||||
function createCommandInterceptor(): {
|
||||
interceptor: CommandInterceptor;
|
||||
handler: (argv: string[], opts: number | { timeoutMs: number; cwd?: string }) => Promise<{ stdout: string; stderr: string; code: number | null; signal: null; killed: false }>;
|
||||
} {
|
||||
const commands: CapturedCommand[] = [];
|
||||
|
||||
const handler = async (
|
||||
argv: string[],
|
||||
optsOrTimeout: number | { timeoutMs: number; cwd?: string },
|
||||
) => {
|
||||
const opts = typeof optsOrTimeout === "number"
|
||||
? { timeoutMs: optsOrTimeout }
|
||||
: optsOrTimeout;
|
||||
|
||||
const captured: CapturedCommand = { argv, opts };
|
||||
|
||||
// Parse gateway agent calls to extract task message
|
||||
if (argv[0] === "openclaw" && argv[1] === "gateway" && argv[2] === "call") {
|
||||
const rpcMethod = argv[3];
|
||||
const paramsIdx = argv.indexOf("--params");
|
||||
if (paramsIdx !== -1 && argv[paramsIdx + 1]) {
|
||||
try {
|
||||
const params = JSON.parse(argv[paramsIdx + 1]);
|
||||
if (rpcMethod === "agent" && params.message) {
|
||||
captured.taskMessage = params.message;
|
||||
}
|
||||
if (rpcMethod === "sessions.patch") {
|
||||
captured.sessionPatch = { key: params.key, model: params.model };
|
||||
}
|
||||
} catch { /* ignore parse errors */ }
|
||||
}
|
||||
}
|
||||
|
||||
commands.push(captured);
|
||||
|
||||
return { stdout: "{}", stderr: "", code: 0, signal: null as null, killed: false as const };
|
||||
};
|
||||
|
||||
const interceptor: CommandInterceptor = {
|
||||
commands,
|
||||
commandsFor(cmd: string) {
|
||||
return commands.filter((c) => c.argv[0] === cmd);
|
||||
},
|
||||
taskMessages() {
|
||||
return commands
|
||||
.filter((c) => c.taskMessage !== undefined)
|
||||
.map((c) => c.taskMessage!);
|
||||
},
|
||||
sessionPatches() {
|
||||
return commands
|
||||
.filter((c) => c.sessionPatch !== undefined)
|
||||
.map((c) => c.sessionPatch!);
|
||||
},
|
||||
reset() {
|
||||
commands.length = 0;
|
||||
},
|
||||
};
|
||||
|
||||
return { interceptor, handler };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test harness
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type TestHarness = {
|
||||
/** Temporary workspace directory. */
|
||||
workspaceDir: string;
|
||||
/** In-memory issue provider. */
|
||||
provider: TestProvider;
|
||||
/** Command interceptor — captures all runCommand calls. */
|
||||
commands: CommandInterceptor;
|
||||
/** The project group ID used for test data. */
|
||||
groupId: string;
|
||||
/** The project data. */
|
||||
project: Project;
|
||||
/** Workflow config. */
|
||||
workflow: WorkflowConfig;
|
||||
/** Write updated projects data to disk. */
|
||||
writeProjects(data: ProjectsData): Promise<void>;
|
||||
/** Read current projects data from disk. */
|
||||
readProjects(): Promise<ProjectsData>;
|
||||
/**
|
||||
* Write a role prompt file to the workspace.
|
||||
* @param role - Role name (e.g. "developer", "tester")
|
||||
* @param content - Prompt file content
|
||||
* @param projectName - If provided, writes project-specific prompt; otherwise writes default.
|
||||
*/
|
||||
writePrompt(role: string, content: string, projectName?: string): Promise<void>;
|
||||
/**
|
||||
* Simulate the agent:bootstrap hook firing for a session key.
|
||||
* Registers the real hook with a mock API, fires it, returns the injected bootstrap files.
|
||||
* This tests the full hook chain: session key → parse → load instructions → inject.
|
||||
*/
|
||||
simulateBootstrap(sessionKey: string): Promise<BootstrapFile[]>;
|
||||
/** Clean up temp directory. */
|
||||
cleanup(): Promise<void>;
|
||||
};
|
||||
|
||||
export type HarnessOptions = {
|
||||
/** Project name (default: "test-project"). */
|
||||
projectName?: string;
|
||||
/** Group ID (default: "-1234567890"). */
|
||||
groupId?: string;
|
||||
/** Repo path (default: "/tmp/test-repo"). */
|
||||
repo?: string;
|
||||
/** Base branch (default: "main"). */
|
||||
baseBranch?: string;
|
||||
/** Workflow config (default: DEFAULT_WORKFLOW). */
|
||||
workflow?: WorkflowConfig;
|
||||
/** Initial worker state overrides. */
|
||||
workers?: Record<string, Partial<import("../projects.js").WorkerState>>;
|
||||
/** Additional projects to seed. */
|
||||
extraProjects?: Record<string, Project>;
|
||||
};
|
||||
|
||||
export async function createTestHarness(opts?: HarnessOptions): Promise<TestHarness> {
|
||||
const {
|
||||
projectName = "test-project",
|
||||
groupId = "-1234567890",
|
||||
repo = "/tmp/test-repo",
|
||||
baseBranch = "main",
|
||||
workflow = DEFAULT_WORKFLOW,
|
||||
workers: workerOverrides,
|
||||
extraProjects,
|
||||
} = opts ?? {};
|
||||
|
||||
// Create temp workspace
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "devclaw-e2e-"));
|
||||
const dataDir = path.join(workspaceDir, "devclaw");
|
||||
const logDir = path.join(dataDir, "log");
|
||||
await fs.mkdir(logDir, { recursive: true });
|
||||
|
||||
// Build project
|
||||
const defaultWorkers: Record<string, import("../projects.js").WorkerState> = {
|
||||
developer: emptyWorkerState(["junior", "medior", "senior"]),
|
||||
tester: emptyWorkerState(["junior", "medior", "senior"]),
|
||||
architect: emptyWorkerState(["junior", "senior"]),
|
||||
};
|
||||
|
||||
// Apply worker overrides
|
||||
if (workerOverrides) {
|
||||
for (const [role, overrides] of Object.entries(workerOverrides)) {
|
||||
if (defaultWorkers[role]) {
|
||||
defaultWorkers[role] = { ...defaultWorkers[role], ...overrides };
|
||||
} else {
|
||||
defaultWorkers[role] = { ...emptyWorkerState([]), ...overrides };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const project: Project = {
|
||||
name: projectName,
|
||||
repo,
|
||||
groupName: "Test Group",
|
||||
deployUrl: "",
|
||||
baseBranch,
|
||||
deployBranch: baseBranch,
|
||||
provider: "github",
|
||||
workers: defaultWorkers,
|
||||
};
|
||||
|
||||
const projectsData: ProjectsData = {
|
||||
projects: {
|
||||
[groupId]: project,
|
||||
...extraProjects,
|
||||
},
|
||||
};
|
||||
|
||||
await writeProjects(workspaceDir, projectsData);
|
||||
|
||||
// Install mock runCommand
|
||||
const { interceptor, handler } = createCommandInterceptor();
|
||||
initRunCommand({
|
||||
runtime: {
|
||||
system: { runCommandWithTimeout: handler },
|
||||
},
|
||||
} as unknown as OpenClawPluginApi);
|
||||
|
||||
// Create test provider
|
||||
const provider = new TestProvider({ workflow });
|
||||
|
||||
return {
|
||||
workspaceDir,
|
||||
provider,
|
||||
commands: interceptor,
|
||||
groupId,
|
||||
project,
|
||||
workflow,
|
||||
async writeProjects(data: ProjectsData) {
|
||||
await writeProjects(workspaceDir, data);
|
||||
},
|
||||
async readProjects() {
|
||||
const { readProjects } = await import("../projects.js");
|
||||
return readProjects(workspaceDir);
|
||||
},
|
||||
async writePrompt(role: string, content: string, forProject?: string) {
|
||||
const dir = forProject
|
||||
? path.join(dataDir, "projects", forProject, "prompts")
|
||||
: path.join(dataDir, "prompts");
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(path.join(dir, `${role}.md`), content, "utf-8");
|
||||
},
|
||||
async simulateBootstrap(sessionKey: string) {
|
||||
// Capture the hook callback by mocking api.registerHook
|
||||
let hookCallback: ((event: any) => Promise<void>) | null = null;
|
||||
const mockApi = {
|
||||
registerHook(_name: string, cb: (event: any) => Promise<void>) {
|
||||
hookCallback = cb;
|
||||
},
|
||||
logger: {
|
||||
info() {},
|
||||
warn() {},
|
||||
error() {},
|
||||
},
|
||||
} as unknown as OpenClawPluginApi;
|
||||
|
||||
registerBootstrapHook(mockApi);
|
||||
if (!hookCallback) throw new Error("registerBootstrapHook did not register a callback");
|
||||
|
||||
// Build a bootstrap event matching what OpenClaw sends
|
||||
const bootstrapFiles: BootstrapFile[] = [];
|
||||
await hookCallback({
|
||||
sessionKey,
|
||||
context: {
|
||||
workspaceDir,
|
||||
bootstrapFiles,
|
||||
},
|
||||
});
|
||||
|
||||
return bootstrapFiles;
|
||||
},
|
||||
async cleanup() {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
16
lib/testing/index.ts
Normal file
16
lib/testing/index.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* testing/ — Test infrastructure for DevClaw integration tests.
|
||||
*
|
||||
* Exports:
|
||||
* - TestProvider: In-memory IssueProvider with call tracking
|
||||
* - createTestHarness: Scaffolds temp workspace + mock runCommand
|
||||
*/
|
||||
export { TestProvider, type ProviderCall } from "./test-provider.js";
|
||||
export {
|
||||
createTestHarness,
|
||||
type TestHarness,
|
||||
type HarnessOptions,
|
||||
type CommandInterceptor,
|
||||
type CapturedCommand,
|
||||
type BootstrapFile,
|
||||
} from "./harness.js";
|
||||
224
lib/testing/test-provider.ts
Normal file
224
lib/testing/test-provider.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* TestProvider — In-memory IssueProvider for integration tests.
|
||||
*
|
||||
* Tracks all method calls for assertion. Issues are stored in a simple map.
|
||||
* No external dependencies — pure TypeScript.
|
||||
*/
|
||||
import type {
|
||||
IssueProvider,
|
||||
Issue,
|
||||
StateLabel,
|
||||
IssueComment,
|
||||
PrStatus,
|
||||
} from "../providers/provider.js";
|
||||
import { getStateLabels } from "../workflow.js";
|
||||
import { DEFAULT_WORKFLOW, type WorkflowConfig } from "../workflow.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Call tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ProviderCall =
|
||||
| { method: "ensureLabel"; args: { name: string; color: string } }
|
||||
| { method: "ensureAllStateLabels"; args: {} }
|
||||
| { method: "createIssue"; args: { title: string; description: string; label: StateLabel; assignees?: string[] } }
|
||||
| { method: "listIssuesByLabel"; args: { label: StateLabel } }
|
||||
| { method: "getIssue"; args: { issueId: number } }
|
||||
| { method: "listComments"; args: { issueId: number } }
|
||||
| { method: "transitionLabel"; args: { issueId: number; from: StateLabel; to: StateLabel } }
|
||||
| { method: "closeIssue"; args: { issueId: number } }
|
||||
| { method: "reopenIssue"; args: { issueId: number } }
|
||||
| { method: "hasMergedMR"; args: { issueId: number } }
|
||||
| { method: "getMergedMRUrl"; args: { issueId: number } }
|
||||
| { method: "getPrStatus"; args: { issueId: number } }
|
||||
| { method: "addComment"; args: { issueId: number; body: string } }
|
||||
| { method: "healthCheck"; args: {} };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TestProvider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class TestProvider implements IssueProvider {
|
||||
/** All issues keyed by iid. */
|
||||
issues = new Map<number, Issue>();
|
||||
/** Comments per issue. */
|
||||
comments = new Map<number, IssueComment[]>();
|
||||
/** Labels that have been ensured. */
|
||||
labels = new Map<string, string>();
|
||||
/** PR status overrides per issue. Default: { state: "closed", url: null }. */
|
||||
prStatuses = new Map<number, PrStatus>();
|
||||
/** Merged MR URLs per issue. */
|
||||
mergedMrUrls = new Map<number, string>();
|
||||
/** All calls, in order. */
|
||||
calls: ProviderCall[] = [];
|
||||
|
||||
private nextIssueId = 1;
|
||||
private workflow: WorkflowConfig;
|
||||
|
||||
constructor(opts?: { workflow?: WorkflowConfig }) {
|
||||
this.workflow = opts?.workflow ?? DEFAULT_WORKFLOW;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Test helpers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/** Create an issue directly in the store (bypasses createIssue tracking). */
|
||||
seedIssue(overrides: Partial<Issue> & { iid: number }): Issue {
|
||||
const issue: Issue = {
|
||||
iid: overrides.iid,
|
||||
title: overrides.title ?? `Issue #${overrides.iid}`,
|
||||
description: overrides.description ?? "",
|
||||
labels: overrides.labels ?? [],
|
||||
state: overrides.state ?? "opened",
|
||||
web_url: overrides.web_url ?? `https://example.com/issues/${overrides.iid}`,
|
||||
};
|
||||
this.issues.set(issue.iid, issue);
|
||||
if (issue.iid >= this.nextIssueId) this.nextIssueId = issue.iid + 1;
|
||||
return issue;
|
||||
}
|
||||
|
||||
/** Set PR status for an issue (used by review pass tests). */
|
||||
setPrStatus(issueId: number, status: PrStatus): void {
|
||||
this.prStatuses.set(issueId, status);
|
||||
}
|
||||
|
||||
/** Get calls filtered by method name. */
|
||||
callsTo<M extends ProviderCall["method"]>(
|
||||
method: M,
|
||||
): Extract<ProviderCall, { method: M }>[] {
|
||||
return this.calls.filter((c) => c.method === method) as any;
|
||||
}
|
||||
|
||||
/** Reset call tracking (keeps issue state). */
|
||||
resetCalls(): void {
|
||||
this.calls = [];
|
||||
}
|
||||
|
||||
/** Full reset — clear everything. */
|
||||
reset(): void {
|
||||
this.issues.clear();
|
||||
this.comments.clear();
|
||||
this.labels.clear();
|
||||
this.prStatuses.clear();
|
||||
this.mergedMrUrls.clear();
|
||||
this.calls = [];
|
||||
this.nextIssueId = 1;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IssueProvider implementation
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
async ensureLabel(name: string, color: string): Promise<void> {
|
||||
this.calls.push({ method: "ensureLabel", args: { name, color } });
|
||||
this.labels.set(name, color);
|
||||
}
|
||||
|
||||
async ensureAllStateLabels(): Promise<void> {
|
||||
this.calls.push({ method: "ensureAllStateLabels", args: {} });
|
||||
const stateLabels = getStateLabels(this.workflow);
|
||||
for (const label of stateLabels) {
|
||||
this.labels.set(label, "#000000");
|
||||
}
|
||||
}
|
||||
|
||||
async createIssue(
|
||||
title: string,
|
||||
description: string,
|
||||
label: StateLabel,
|
||||
assignees?: string[],
|
||||
): Promise<Issue> {
|
||||
this.calls.push({ method: "createIssue", args: { title, description, label, assignees } });
|
||||
const iid = this.nextIssueId++;
|
||||
const issue: Issue = {
|
||||
iid,
|
||||
title,
|
||||
description,
|
||||
labels: [label],
|
||||
state: "opened",
|
||||
web_url: `https://example.com/issues/${iid}`,
|
||||
};
|
||||
this.issues.set(iid, issue);
|
||||
return issue;
|
||||
}
|
||||
|
||||
async listIssuesByLabel(label: StateLabel): Promise<Issue[]> {
|
||||
this.calls.push({ method: "listIssuesByLabel", args: { label } });
|
||||
return [...this.issues.values()].filter((i) => i.labels.includes(label));
|
||||
}
|
||||
|
||||
async getIssue(issueId: number): Promise<Issue> {
|
||||
this.calls.push({ method: "getIssue", args: { issueId } });
|
||||
const issue = this.issues.get(issueId);
|
||||
if (!issue) throw new Error(`Issue #${issueId} not found in TestProvider`);
|
||||
return issue;
|
||||
}
|
||||
|
||||
async listComments(issueId: number): Promise<IssueComment[]> {
|
||||
this.calls.push({ method: "listComments", args: { issueId } });
|
||||
return this.comments.get(issueId) ?? [];
|
||||
}
|
||||
|
||||
async transitionLabel(
|
||||
issueId: number,
|
||||
from: StateLabel,
|
||||
to: StateLabel,
|
||||
): Promise<void> {
|
||||
this.calls.push({ method: "transitionLabel", args: { issueId, from, to } });
|
||||
const issue = this.issues.get(issueId);
|
||||
if (!issue) throw new Error(`Issue #${issueId} not found in TestProvider`);
|
||||
// Remove all state labels, add the new one
|
||||
const stateLabels = getStateLabels(this.workflow);
|
||||
issue.labels = issue.labels.filter((l) => !stateLabels.includes(l));
|
||||
issue.labels.push(to);
|
||||
}
|
||||
|
||||
async closeIssue(issueId: number): Promise<void> {
|
||||
this.calls.push({ method: "closeIssue", args: { issueId } });
|
||||
const issue = this.issues.get(issueId);
|
||||
if (issue) issue.state = "closed";
|
||||
}
|
||||
|
||||
async reopenIssue(issueId: number): Promise<void> {
|
||||
this.calls.push({ method: "reopenIssue", args: { issueId } });
|
||||
const issue = this.issues.get(issueId);
|
||||
if (issue) issue.state = "opened";
|
||||
}
|
||||
|
||||
hasStateLabel(issue: Issue, expected: StateLabel): boolean {
|
||||
return issue.labels.includes(expected);
|
||||
}
|
||||
|
||||
getCurrentStateLabel(issue: Issue): StateLabel | null {
|
||||
const stateLabels = getStateLabels(this.workflow);
|
||||
return stateLabels.find((l) => issue.labels.includes(l)) ?? null;
|
||||
}
|
||||
|
||||
async hasMergedMR(issueId: number): Promise<boolean> {
|
||||
this.calls.push({ method: "hasMergedMR", args: { issueId } });
|
||||
return this.mergedMrUrls.has(issueId);
|
||||
}
|
||||
|
||||
async getMergedMRUrl(issueId: number): Promise<string | null> {
|
||||
this.calls.push({ method: "getMergedMRUrl", args: { issueId } });
|
||||
return this.mergedMrUrls.get(issueId) ?? null;
|
||||
}
|
||||
|
||||
async getPrStatus(issueId: number): Promise<PrStatus> {
|
||||
this.calls.push({ method: "getPrStatus", args: { issueId } });
|
||||
return this.prStatuses.get(issueId) ?? { state: "closed", url: null };
|
||||
}
|
||||
|
||||
async addComment(issueId: number, body: string): Promise<void> {
|
||||
this.calls.push({ method: "addComment", args: { issueId, body } });
|
||||
const existing = this.comments.get(issueId) ?? [];
|
||||
existing.push({ author: "test", body, created_at: new Date().toISOString() });
|
||||
this.comments.set(issueId, existing);
|
||||
}
|
||||
|
||||
async healthCheck(): Promise<boolean> {
|
||||
this.calls.push({ method: "healthCheck", args: {} });
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user