Files
harness-tools/extensions/workflow/profiles.ts
T

815 lines
24 KiB
TypeScript

import { createHash } from "node:crypto";
import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
import * as path from "node:path";
import { getAgentDir } from "@earendil-works/pi-coding-agent";
import {
PROFILE_ROLES,
REASONING_EFFORTS,
type AgentProfile,
type AssignmentStageKind,
type EligibilityResult,
type ProfileCatalog,
type ProfileDiagnostic,
type ProfileRole,
type ProfileSelection,
type StandardAgentConfig,
type WorkflowFacts,
} from "./domain.ts";
const PROFILE_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const TOOL_NAME = /^[a-zA-Z0-9_-]+$/;
const TOP_LEVEL_FIELDS = new Set([
"name",
"description",
"roles",
"model",
"provider",
"effort",
"tools",
"selection",
]);
const SELECTION_FIELDS = new Set([
"stages",
"anyTags",
"allTags",
"pathGlobs",
"keywords",
]);
interface ParsedFrontmatter {
values: Record<string, unknown>;
body: string;
}
interface LoadSourceResult {
profiles: AgentProfile[];
diagnostics: ProfileDiagnostic[];
}
type ProfileSourceMode = "workflow" | "standard" | "project";
export interface LoadAgentProfilesOptions {
cwd: string;
projectTrusted: boolean;
agentDir?: string;
bundledRoot?: string;
standardPackageRoot?: string;
projectRoot?: string;
standardAgents?: StandardAgentConfig;
}
function reason(error: unknown) {
return error instanceof Error ? error.message : String(error);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function compareText(left: string, right: string) {
if (left === right) return 0;
return left < right ? -1 : 1;
}
function isEscaped(value: string, index: number) {
let backslashes = 0;
for (
let cursor = index - 1;
cursor >= 0 && value[cursor] === "\\";
cursor--
) {
backslashes++;
}
return backslashes % 2 === 1;
}
function stripComment(value: string) {
let quote: "'" | '"' | undefined;
for (let index = 0; index < value.length; index++) {
const character = value[index];
if (quote) {
if (quote === "'" && character === "'" && value[index + 1] === "'") {
index++;
} else if (
character === quote &&
(quote === "'" || !isEscaped(value, index))
) {
quote = undefined;
}
continue;
}
if (character === "'" || character === '"') {
quote = character;
} else if (
character === "#" &&
(index === 0 || /\s/.test(value[index - 1] ?? ""))
) {
return value.slice(0, index).trimEnd();
}
}
return value;
}
function splitFlowItems(value: string, label: string) {
const items: string[] = [];
let quote: "'" | '"' | undefined;
let start = 0;
for (let index = 0; index < value.length; index++) {
const character = value[index];
if (quote) {
if (quote === "'" && character === "'" && value[index + 1] === "'") {
index++;
} else if (
character === quote &&
(quote === "'" || !isEscaped(value, index))
) {
quote = undefined;
}
} else if (character === "'" || character === '"') {
quote = character;
} else if (character === ",") {
items.push(value.slice(start, index));
start = index + 1;
}
}
if (quote) throw new Error(`${label}: unterminated quoted value`);
items.push(value.slice(start));
return items;
}
function scalar(value: string, label: string) {
const trimmed = stripComment(value).trim();
if (trimmed.length === 0) throw new Error(`${label}: value cannot be empty`);
if (trimmed.startsWith('"')) {
try {
const parsed = JSON.parse(trimmed) as unknown;
if (typeof parsed !== "string") throw new Error("not a string");
return parsed;
} catch (error) {
throw new Error(`${label}: invalid quoted string`, { cause: error });
}
}
if (trimmed.startsWith("'")) {
if (!trimmed.endsWith("'") || trimmed.length < 2) {
throw new Error(`${label}: invalid quoted string`);
}
return trimmed.slice(1, -1).replaceAll("''", "'");
}
return trimmed;
}
function yamlValue(value: string, label: string): string | string[] {
const trimmed = stripComment(value).trim();
if (!trimmed.startsWith("[")) return scalar(trimmed, label);
if (!trimmed.endsWith("]")) throw new Error(`${label}: invalid flow array`);
const inside = trimmed.slice(1, -1).trim();
if (inside.length === 0) return [];
return splitFlowItems(inside, label).map((entry, index) =>
scalar(entry, `${label}[${index}]`),
);
}
function parseFrontmatter(
contents: string,
sourcePath: string,
): ParsedFrontmatter {
const normalized = contents.replaceAll("\r\n", "\n");
const lines = normalized.split("\n");
if (lines[0] !== "---") throw new Error("missing YAML frontmatter");
const closing = lines.indexOf("---", 1);
if (closing < 0) throw new Error("unterminated YAML frontmatter");
const values: Record<string, unknown> = {};
let section: Record<string, unknown> | undefined;
let sectionName: string | undefined;
let skippedUnknownIndent: number | undefined;
let pendingList:
| { container: Record<string, unknown>; key: string; indent: number }
| undefined;
for (let lineIndex = 1; lineIndex < closing; lineIndex++) {
const raw = lines[lineIndex] ?? "";
if (raw.trim().length === 0 || raw.trimStart().startsWith("#")) continue;
if (raw.includes("\t"))
throw new Error(
`frontmatter line ${lineIndex + 1}: tabs are not supported`,
);
const indent = raw.length - raw.trimStart().length;
if (skippedUnknownIndent !== undefined) {
if (indent > skippedUnknownIndent) continue;
skippedUnknownIndent = undefined;
}
if (indent % 2 !== 0) {
throw new Error(
`frontmatter line ${lineIndex + 1}: indentation must use pairs of spaces`,
);
}
const text = stripComment(raw.trim());
if (text.length === 0) continue;
if (text.startsWith("- ")) {
if (!pendingList || indent !== pendingList.indent + 2) {
throw new Error(
`frontmatter line ${lineIndex + 1}: unexpected list item`,
);
}
const list = pendingList.container[pendingList.key];
if (!Array.isArray(list))
throw new Error(`${sourcePath}: invalid list state`);
list.push(scalar(text.slice(2), `frontmatter line ${lineIndex + 1}`));
continue;
}
pendingList = undefined;
const separator = text.indexOf(":");
if (separator < 1)
throw new Error(`frontmatter line ${lineIndex + 1}: expected key: value`);
const key = text.slice(0, separator).trim();
const rawValue = text.slice(separator + 1).trim();
if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(key)) {
throw new Error(
`frontmatter line ${lineIndex + 1}: invalid key ${JSON.stringify(key)}`,
);
}
let container: Record<string, unknown>;
if (indent === 0) {
container = values;
section = undefined;
sectionName = undefined;
} else if (indent === 2 && section && sectionName) {
container = section;
} else {
throw new Error(`frontmatter line ${lineIndex + 1}: unsupported nesting`);
}
if (Object.hasOwn(container, key)) {
throw new Error(
`frontmatter line ${lineIndex + 1}: duplicate field ${key}`,
);
}
if (rawValue.length > 0) {
container[key] = yamlValue(rawValue, `frontmatter field ${key}`);
continue;
}
const knownField =
indent === 0 ? TOP_LEVEL_FIELDS.has(key) : SELECTION_FIELDS.has(key);
if (!knownField) {
container[key] = "<ignored nested value>";
skippedUnknownIndent = indent;
continue;
}
if (indent === 0 && key === "selection") {
section = {};
sectionName = key;
values[key] = section;
} else {
container[key] = [];
pendingList = { container, key, indent };
}
}
return {
values,
body: lines
.slice(closing + 1)
.join("\n")
.trim(),
};
}
function requiredString(value: unknown, field: string) {
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(`${field} must be a non-empty string`);
}
return value.trim();
}
function optionalString(value: unknown, field: string) {
return value === undefined ? undefined : requiredString(value, field);
}
function stringList(value: unknown, field: string, allowCsv = false) {
const list =
allowCsv && typeof value === "string"
? value.split(",").map((entry) => entry.trim())
: value;
if (!Array.isArray(list) || list.some((entry) => typeof entry !== "string")) {
throw new Error(
`${field} must be an array of strings${allowCsv ? " or CSV string" : ""}`,
);
}
const normalized = list.map((entry) => entry.trim());
if (normalized.some((entry) => entry.length === 0)) {
throw new Error(`${field} cannot contain empty values`);
}
if (new Set(normalized).size !== normalized.length) {
throw new Error(`${field} cannot contain duplicate values`);
}
return normalized;
}
function enumList<T extends string>(
value: unknown,
field: string,
allowed: readonly T[],
): T[] {
const entries = stringList(value, field);
const invalid = entries.filter((entry) => !allowed.includes(entry as T));
if (invalid.length > 0) {
throw new Error(
`${field} contains unsupported values: ${invalid.join(", ")}`,
);
}
return entries as T[];
}
function parseSelection(value: unknown) {
if (!isRecord(value)) throw new Error("selection must be an object");
const warnings = Object.keys(value)
.filter((key) => !SELECTION_FIELDS.has(key))
.map((key) => `unknown selection field ${key}`);
const stages = enumList(value.stages, "selection.stages", PROFILE_ROLES);
if (stages.length === 0)
throw new Error("selection.stages must not be empty");
const normalize = (entries: string[]) =>
entries.map((entry) => entry.toLowerCase());
const selection: ProfileSelection = {
stages,
anyTags: normalize(stringList(value.anyTags ?? [], "selection.anyTags")),
allTags: normalize(stringList(value.allTags ?? [], "selection.allTags")),
pathGlobs: stringList(value.pathGlobs ?? [], "selection.pathGlobs").map(
(entry) => entry.replaceAll("\\", "/"),
),
keywords: normalize(stringList(value.keywords ?? [], "selection.keywords")),
};
return { selection, warnings };
}
function buildStandardProfile(
parsed: ParsedFrontmatter,
contents: string,
sourcePath: string,
resolvedPath: string,
source: AgentProfile["source"],
fallbackName: string,
roles: ProfileRole[],
) {
const name = optionalString(parsed.values.name, "name") ?? fallbackName;
if (!PROFILE_NAME.test(name)) {
throw new Error("name must be lowercase kebab case");
}
const description =
optionalString(parsed.values.description, "description") ?? name;
const model = optionalString(parsed.values.model, "model");
const effort = optionalString(parsed.values.thinking, "thinking");
if (effort && !REASONING_EFFORTS.includes(effort as never)) {
throw new Error(`thinking must be one of ${REASONING_EFFORTS.join(", ")}`);
}
const tools =
parsed.values.tools === undefined
? undefined
: stringList(parsed.values.tools, "tools", true);
if (tools?.some((tool) => !TOOL_NAME.test(tool))) {
throw new Error("tools contains an invalid tool name");
}
if (parsed.body.length === 0) {
throw new Error("agent instructions must not be empty");
}
const selection: ProfileSelection = {
stages: [...roles],
anyTags: [],
allTags: [],
pathGlobs: [],
keywords: [],
};
const digestInput = `${contents}\nworkflow-role-overlay:${JSON.stringify(roles)}`;
const profile: AgentProfile = {
name,
description,
roles: [...roles],
...(model ? { model } : {}),
...(effort ? { effort: effort as AgentProfile["effort"] } : {}),
...(tools ? { tools } : {}),
selection,
instructions: parsed.body,
source,
sourcePath,
resolvedPath,
digest: createHash("sha256").update(digestInput).digest("hex"),
};
return { profile, warnings: [] as string[] };
}
function buildProfile(
contents: string,
sourcePath: string,
resolvedPath: string,
source: AgentProfile["source"],
) {
const parsed = parseFrontmatter(contents, sourcePath);
const warnings = Object.keys(parsed.values)
.filter((key) => !TOP_LEVEL_FIELDS.has(key))
.map((key) => `unknown profile field ${key}`);
const name = requiredString(parsed.values.name, "name");
if (!PROFILE_NAME.test(name)) {
throw new Error("name must be lowercase kebab case");
}
const description = requiredString(parsed.values.description, "description");
const roles = enumList(
parsed.values.roles,
"roles",
PROFILE_ROLES,
) as ProfileRole[];
if (roles.length === 0) throw new Error("roles must not be empty");
const model = optionalString(parsed.values.model, "model");
const provider = optionalString(parsed.values.provider, "provider");
if (provider && (!model || model.includes("/"))) {
throw new Error("provider requires a bare model field");
}
const effort = optionalString(parsed.values.effort, "effort");
if (effort && !REASONING_EFFORTS.includes(effort as never)) {
throw new Error(`effort must be one of ${REASONING_EFFORTS.join(", ")}`);
}
const tools =
parsed.values.tools === undefined
? undefined
: stringList(parsed.values.tools, "tools", true);
if (tools?.some((tool) => !TOOL_NAME.test(tool))) {
throw new Error("tools contains an invalid tool name");
}
if (parsed.body.length === 0)
throw new Error("agent instructions must not be empty");
const { selection, warnings: selectionWarnings } = parseSelection(
parsed.values.selection,
);
const profile: AgentProfile = {
name,
description,
roles,
...(model ? { model } : {}),
...(provider ? { provider } : {}),
...(effort ? { effort: effort as AgentProfile["effort"] } : {}),
...(tools ? { tools } : {}),
selection,
instructions: parsed.body,
source,
sourcePath,
resolvedPath,
digest: createHash("sha256").update(contents).digest("hex"),
};
return { profile, warnings: [...warnings, ...selectionWarnings] };
}
function isInside(parent: string, child: string) {
const relative = path.relative(parent, child);
return (
relative === "" ||
(!relative.startsWith("..") && !path.isAbsolute(relative))
);
}
async function loadSource(
root: string,
source: AgentProfile["source"],
required: boolean,
mode: ProfileSourceMode = "workflow",
standardRoles?: Record<string, ProfileRole[]>,
): Promise<LoadSourceResult> {
const diagnostics: ProfileDiagnostic[] = [];
let resolvedRoot: string;
try {
const rootInfo = await lstat(root);
if (rootInfo.isSymbolicLink()) {
throw new Error("profile root must not be a symlink");
}
if (!rootInfo.isDirectory()) {
throw new Error("profile root must be a directory");
}
resolvedRoot = await realpath(root);
} catch (error) {
if (!required && (error as NodeJS.ErrnoException).code === "ENOENT") {
return { profiles: [], diagnostics };
}
throw new Error(
`${root}: unable to resolve profile root: ${reason(error)}`,
{
cause: error,
},
);
}
let entries;
try {
entries = await readdir(root, { withFileTypes: true });
} catch (error) {
throw new Error(`${root}: unable to read profile root`, { cause: error });
}
const profiles: AgentProfile[] = [];
const names = new Set<string>();
for (const entry of entries.sort((left, right) =>
compareText(left.name, right.name),
)) {
if (
!entry.name.endsWith(".md") ||
(!entry.isFile() && !entry.isSymbolicLink())
)
continue;
const sourcePath = path.join(root, entry.name);
try {
const resolvedPath = await realpath(sourcePath);
if (!isInside(resolvedRoot, resolvedPath)) {
throw new Error("resolved path escapes the profile root");
}
if (!(await stat(resolvedPath)).isFile()) {
throw new Error("profile is not a regular file");
}
const contents = await readFile(resolvedPath, "utf8");
const parsed = parseFrontmatter(contents, sourcePath);
const declaredName =
optionalString(parsed.values.name, "name") ??
entry.name.replace(/\.md$/, "");
const isStandardDefinition =
mode === "standard" ||
(mode === "project" && parsed.values.roles === undefined);
let built;
if (isStandardDefinition) {
const roles = standardRoles?.[declaredName];
if (!roles) continue;
built = buildStandardProfile(
parsed,
contents,
sourcePath,
resolvedPath,
source === "project" ? "standard-project" : source,
entry.name.replace(/\.md$/, ""),
roles,
);
} else {
built = buildProfile(contents, sourcePath, resolvedPath, source);
}
const { profile, warnings } = built;
for (const message of warnings) {
diagnostics.push({ sourcePath, severity: "warning", message });
}
if (names.has(profile.name)) {
diagnostics.push({
sourcePath,
severity: "error",
message: `duplicate profile name ${profile.name}; first lexical file wins`,
});
continue;
}
names.add(profile.name);
profiles.push(profile);
} catch (error) {
diagnostics.push({
sourcePath,
severity: "error",
message: reason(error),
});
}
}
return { profiles, diagnostics };
}
async function assertProjectProfileAncestry(root: string) {
const parent = path.dirname(root);
const parentInfo = await lstat(parent);
if (parentInfo.isSymbolicLink()) {
throw new Error(
`${parent}: project profile ancestry must not be a symlink`,
);
}
if (!parentInfo.isDirectory()) {
throw new Error(`${parent}: project profile parent must be a directory`);
}
}
async function findNearestProjectRoot(cwd: string) {
let current = path.resolve(cwd);
while (true) {
const candidate = path.join(current, ".pi", "agents");
try {
await lstat(candidate);
return candidate;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw new Error(`${candidate}: unable to inspect profile root`, {
cause: error,
});
}
}
const parent = path.dirname(current);
if (parent === current) return undefined;
current = parent;
}
}
export async function loadAgentProfiles(
options: LoadAgentProfilesOptions,
): Promise<ProfileCatalog> {
const bundledRoot =
options.bundledRoot ?? path.join(import.meta.dirname, "agents");
const bundled = await loadSource(bundledRoot, "bundled", true);
const standardAgents = options.standardAgents ?? {
enabled: false,
roles: {},
} satisfies StandardAgentConfig;
const agentDir = options.agentDir ?? getAgentDir();
const standardPackageRoot =
options.standardPackageRoot ??
path.join(
agentDir,
"extensions",
"pi-interactive-subagents",
"agents",
);
const standardPackage = standardAgents.enabled
? await loadSource(
standardPackageRoot,
"standard-package",
false,
"standard",
standardAgents.roles,
)
: { profiles: [], diagnostics: [] };
const standardGlobal = standardAgents.enabled
? await loadSource(
path.join(agentDir, "agents"),
"standard-global",
false,
"standard",
standardAgents.roles,
)
: { profiles: [], diagnostics: [] };
const projectRoot = options.projectTrusted
? (options.projectRoot ?? (await findNearestProjectRoot(options.cwd)))
: undefined;
if (projectRoot) await assertProjectProfileAncestry(projectRoot);
const project = projectRoot
? await loadSource(
projectRoot,
"project",
false,
"project",
standardAgents.enabled ? standardAgents.roles : undefined,
)
: { profiles: [], diagnostics: [] };
const byName = new Map(
bundled.profiles.map((profile) => [profile.name, profile]),
);
for (const profile of standardPackage.profiles) {
byName.set(profile.name, profile);
}
for (const profile of standardGlobal.profiles) {
byName.set(profile.name, profile);
}
for (const profile of project.profiles) {
byName.set(profile.name, profile);
}
return {
profiles: [...byName.values()].sort((left, right) =>
compareText(left.name, right.name),
),
diagnostics: [
...bundled.diagnostics,
...standardPackage.diagnostics,
...standardGlobal.diagnostics,
...project.diagnostics,
],
};
}
function globPattern(pattern: string) {
let expression = "^";
for (let index = 0; index < pattern.length; index++) {
const character = pattern[index] ?? "";
if (character === "*" && pattern[index + 1] === "*") {
if (pattern[index + 2] === "/") {
expression += "(?:.*/)?";
index += 2;
} else {
expression += ".*";
index++;
}
} else if (character === "*") {
expression += "[^/]*";
} else if (character === "?") {
expression += "[^/]";
} else {
expression += character.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
}
}
return new RegExp(`${expression}$`);
}
export function evaluateProfileEligibility(
profile: AgentProfile,
facts: WorkflowFacts,
role: ProfileRole,
): EligibilityResult {
const reasons: string[] = [];
const tags = new Set(facts.tags.map((tag) => tag.toLowerCase()));
const terms = new Set(facts.requestTerms.map((term) => term.toLowerCase()));
const paths = facts.candidatePaths.map((candidate) =>
candidate.replaceAll("\\", "/"),
);
if (!profile.roles.includes(role))
reasons.push(`does not support role ${role}`);
if (!profile.selection.stages.includes(facts.stage)) {
reasons.push(`does not support stage ${facts.stage}`);
}
const missingTags = profile.selection.allTags.filter((tag) => !tags.has(tag));
if (missingTags.length > 0)
reasons.push(`missing required tags: ${missingTags.join(", ")}`);
if (
profile.selection.anyTags.length > 0 &&
!profile.selection.anyTags.some((tag) => tags.has(tag))
) {
reasons.push("no optional tag matched");
}
if (
profile.selection.pathGlobs.length > 0 &&
!profile.selection.pathGlobs.some((pattern) => {
const matcher = globPattern(pattern);
return paths.some((candidate) => matcher.test(candidate));
})
) {
reasons.push("no candidate path matched");
}
if (
profile.selection.keywords.length > 0 &&
!profile.selection.keywords.some((keyword) => terms.has(keyword))
) {
reasons.push("no request keyword matched");
}
return {
profile,
eligible: reasons.length === 0,
reasons,
specificity:
profile.selection.allTags.length +
profile.selection.anyTags.length +
profile.selection.pathGlobs.length +
profile.selection.keywords.length,
};
}
export function eligibleProfiles(
profiles: AgentProfile[],
facts: WorkflowFacts,
role: ProfileRole,
) {
const evaluations = profiles.map((profile) =>
evaluateProfileEligibility(profile, facts, role),
);
const hasEligibleStandard = evaluations.some(
(evaluation) =>
evaluation.eligible && evaluation.profile.source.startsWith("standard-"),
);
const ranked = hasEligibleStandard
? evaluations.map((evaluation) =>
evaluation.eligible && evaluation.profile.source === "bundled"
? {
...evaluation,
eligible: false,
reasons: [
...evaluation.reasons,
`eligible standard agent available for role ${role}`,
],
}
: evaluation,
)
: evaluations;
return ranked.sort(
(left, right) =>
Number(right.eligible) - Number(left.eligible) ||
right.specificity - left.specificity ||
compareText(left.profile.name, right.profile.name),
);
}
export function selectInitialDiscoveryProfile(
profiles: AgentProfile[],
facts: Omit<WorkflowFacts, "stage">,
) {
const evaluations = eligibleProfiles(
profiles,
{ ...facts, stage: "discover" as AssignmentStageKind },
"discover",
);
const selected = evaluations.find((evaluation) => evaluation.eligible);
if (!selected)
throw new Error("No eligible discovery Agent Profile is available");
return { selected: selected.profile, evaluations };
}