139 lines
4.2 KiB
TypeScript
139 lines
4.2 KiB
TypeScript
/**
|
|
* llm-model-selector.ts — LLM-powered intelligent model selection.
|
|
*
|
|
* Uses an LLM to understand model capabilities and assign optimal models to DevClaw roles.
|
|
*/
|
|
import { runCommand } from "../run-command.js";
|
|
|
|
export type ModelAssignment = {
|
|
dev: {
|
|
junior: string;
|
|
medior: string;
|
|
senior: string;
|
|
};
|
|
qa: {
|
|
reviewer: string;
|
|
tester: string;
|
|
};
|
|
};
|
|
|
|
/**
|
|
* Use an LLM to intelligently select and assign models to DevClaw roles.
|
|
*/
|
|
export async function selectModelsWithLLM(
|
|
availableModels: Array<{ model: string; provider: string }>,
|
|
sessionKey?: string,
|
|
): Promise<ModelAssignment | null> {
|
|
if (availableModels.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
// If only one model, assign it to all roles
|
|
if (availableModels.length === 1) {
|
|
const model = availableModels[0].model;
|
|
return {
|
|
dev: { junior: model, medior: model, senior: model },
|
|
qa: { reviewer: model, tester: model },
|
|
};
|
|
}
|
|
|
|
// Create a prompt for the LLM
|
|
const modelList = availableModels.map((m) => m.model).join("\n");
|
|
|
|
const prompt = `You are an AI model expert. Analyze the following authenticated AI models and assign them to DevClaw development roles based on their capabilities.
|
|
|
|
Available models:
|
|
${modelList}
|
|
|
|
Assign models to these roles based on capability:
|
|
- **senior** (most capable): Complex architecture, refactoring, critical decisions
|
|
- **medior** (balanced): Features, bug fixes, code review
|
|
- **junior** (fast/efficient): Simple fixes, testing, routine tasks
|
|
- **reviewer** (same as medior): Code review
|
|
- **tester** (same as junior): Testing
|
|
|
|
Rules:
|
|
1. Prefer same provider for consistency
|
|
2. Assign most capable model to senior
|
|
3. Assign mid-tier model to medior/reviewer
|
|
4. Assign fastest/cheapest model to junior/tester
|
|
5. Consider model version numbers (higher = newer/better)
|
|
6. Stable versions (no date) > snapshot versions (with date like 20250514)
|
|
|
|
Return ONLY a JSON object in this exact format (no markdown, no explanation):
|
|
{
|
|
"dev": {
|
|
"junior": "provider/model-name",
|
|
"medior": "provider/model-name",
|
|
"senior": "provider/model-name"
|
|
},
|
|
"qa": {
|
|
"reviewer": "provider/model-name",
|
|
"tester": "provider/model-name"
|
|
}
|
|
}`;
|
|
|
|
try {
|
|
const sessionId = sessionKey ?? "devclaw-model-selection";
|
|
|
|
const result = await runCommand(
|
|
["openclaw", "agent", "--local", "--session-id", sessionId, "--message", prompt, "--json"],
|
|
{ timeoutMs: 30_000 },
|
|
);
|
|
|
|
const output = result.stdout.trim();
|
|
|
|
// Parse the response from openclaw agent --json
|
|
const lines = output.split("\n");
|
|
const jsonStartIndex = lines.findIndex((line) => line.trim().startsWith("{"));
|
|
|
|
if (jsonStartIndex === -1) {
|
|
throw new Error("No JSON found in LLM response");
|
|
}
|
|
|
|
const jsonString = lines.slice(jsonStartIndex).join("\n");
|
|
|
|
// openclaw agent --json returns: { payloads: [{ text: "```json\n{...}\n```" }], meta: {...} }
|
|
const response = JSON.parse(jsonString);
|
|
|
|
if (!response.payloads || !Array.isArray(response.payloads) || response.payloads.length === 0) {
|
|
throw new Error("Invalid openclaw agent response structure - missing payloads");
|
|
}
|
|
|
|
// Extract text from first payload
|
|
const textContent = response.payloads[0].text;
|
|
if (!textContent) {
|
|
throw new Error("Empty text content in openclaw agent payload");
|
|
}
|
|
|
|
// Strip markdown code blocks (```json and ```)
|
|
const cleanJson = textContent
|
|
.replace(/```json\n?/g, '')
|
|
.replace(/```\n?/g, '')
|
|
.trim();
|
|
|
|
// Parse the actual model assignment JSON
|
|
const assignment = JSON.parse(cleanJson);
|
|
|
|
// Log what we got for debugging
|
|
console.log("LLM returned:", JSON.stringify(assignment, null, 2));
|
|
|
|
// Validate the structure
|
|
if (
|
|
!assignment.dev?.junior ||
|
|
!assignment.dev?.medior ||
|
|
!assignment.dev?.senior ||
|
|
!assignment.qa?.reviewer ||
|
|
!assignment.qa?.tester
|
|
) {
|
|
console.error("Invalid assignment structure. Got:", assignment);
|
|
throw new Error(`Invalid assignment structure from LLM. Missing fields in: ${JSON.stringify(Object.keys(assignment))}`);
|
|
}
|
|
|
|
return assignment as ModelAssignment;
|
|
} catch (err) {
|
|
console.error("LLM model selection failed:", (err as Error).message);
|
|
return null;
|
|
}
|
|
}
|