Add ISAC mechanic gap handoff actions
This commit is contained in:
JB
@@ -0,0 +1,151 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
generated: true,
|
||||
existing: false,
|
||||
recipients: [{ id: 'admin-1' }],
|
||||
notifications: [] as Array<Record<string, unknown>>,
|
||||
audits: [] as Array<Record<string, unknown>>
|
||||
}));
|
||||
|
||||
vi.mock('$lib/server/database', () => ({
|
||||
getDatabase: async () => ({
|
||||
prepare: (sql: string) => ({
|
||||
get: async () => sql.includes("action='mechanic_ai.generated'")
|
||||
? (mocks.generated ? { id: 'generation-audit' } : undefined)
|
||||
: sql.includes("action='mechanic_gap.submitted'")
|
||||
? (mocks.existing ? { id: 'existing-report' } : undefined)
|
||||
: undefined,
|
||||
all: async () => sql.includes('SELECT DISTINCT users.id') ? mocks.recipients : []
|
||||
})
|
||||
})
|
||||
}));
|
||||
vi.mock('$lib/server/notifications', () => ({
|
||||
createNotification: async (input: Record<string, unknown>) => {
|
||||
mocks.notifications.push(input);
|
||||
return `notification-${mocks.notifications.length}`;
|
||||
}
|
||||
}));
|
||||
vi.mock('$lib/server/audit', () => ({
|
||||
recordIsacAudit: async (input: Record<string, unknown>) => {
|
||||
mocks.audits.push(input);
|
||||
}
|
||||
}));
|
||||
|
||||
import {
|
||||
MechanicGapReportError,
|
||||
parseMechanicGapReport,
|
||||
submitMechanicGapReport
|
||||
} from './mechanic-gap-report';
|
||||
|
||||
const report = {
|
||||
context: {
|
||||
sourceKind: 'talent',
|
||||
sourceName: 'Chain Reaction',
|
||||
sourceDescription: 'Every third hit transfers the mark.'
|
||||
},
|
||||
valueColumns: [
|
||||
{ key: 'tierI', variantKey: 'I', label: 'TIER I' },
|
||||
{ key: 'tierII', variantKey: 'II', label: 'TIER II' }
|
||||
],
|
||||
allowAssignedAbility: false,
|
||||
allowProtocolUpgradeScaling: false,
|
||||
transcript: [],
|
||||
response: {
|
||||
status: 'capability_gap',
|
||||
summary: 'The runtime cannot select a transfer target.',
|
||||
assumptions: [],
|
||||
warnings: [],
|
||||
gaps: [{
|
||||
area: 'runtime',
|
||||
requiredBehavior: 'Select the nearest eligible enemy.',
|
||||
whyUnsupported: 'No target selector is available.',
|
||||
developerChange: 'Add a deterministic nearby-target selector.',
|
||||
workaround: null
|
||||
}],
|
||||
meta: {
|
||||
requestId: 'req_gap_123',
|
||||
model: 'test-model',
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
totalTokens: 150
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
describe('mechanic gap report validation', () => {
|
||||
beforeEach(() => {
|
||||
mocks.generated = true;
|
||||
mocks.existing = false;
|
||||
mocks.recipients = [{ id: 'admin-1' }];
|
||||
mocks.notifications.length = 0;
|
||||
mocks.audits.length = 0;
|
||||
});
|
||||
|
||||
it('accepts a bounded capability-gap handoff that matches the source profile', () => {
|
||||
expect(parseMechanicGapReport(report)).toMatchObject({
|
||||
context: report.context,
|
||||
response: {
|
||||
status: 'capability_gap',
|
||||
gaps: [{ area: 'runtime' }]
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a non-gap assistant response', () => {
|
||||
expect(() => parseMechanicGapReport({
|
||||
...report,
|
||||
response: { ...report.response, status: 'draft_ready' }
|
||||
})).toThrow(MechanicGapReportError);
|
||||
});
|
||||
|
||||
it('rejects client-selected capabilities that do not match the source profile', () => {
|
||||
expect(() => parseMechanicGapReport({
|
||||
...report,
|
||||
allowAssignedAbility: true
|
||||
})).toThrow('Assigned-ability scope does not match the mechanic source.');
|
||||
});
|
||||
|
||||
it('rejects unbounded source wording', () => {
|
||||
expect(() => parseMechanicGapReport({
|
||||
...report,
|
||||
context: { ...report.context, sourceDescription: 'x'.repeat(12_001) }
|
||||
})).toThrow('Source wording is too long.');
|
||||
});
|
||||
|
||||
it('notifies active administrators and records complete audit evidence after submission', async () => {
|
||||
const result = await submitMechanicGapReport('author-1', parseMechanicGapReport(report));
|
||||
|
||||
expect(result).toEqual({ alreadySubmitted: false, recipientCount: 1 });
|
||||
expect(mocks.notifications).toEqual([
|
||||
expect.objectContaining({
|
||||
userId: 'admin-1',
|
||||
category: 'isac',
|
||||
mandatory: true
|
||||
})
|
||||
]);
|
||||
expect(mocks.audits).toEqual([
|
||||
expect.objectContaining({
|
||||
actorId: 'author-1',
|
||||
category: 'ai_assistance',
|
||||
action: 'mechanic_gap.submitted',
|
||||
entityId: 'mechanic_gap:req_gap_123',
|
||||
after: expect.objectContaining({
|
||||
requestId: 'req_gap_123',
|
||||
sourceKind: 'talent',
|
||||
recipientCount: 1,
|
||||
gaps: report.response.gaps
|
||||
})
|
||||
})
|
||||
]);
|
||||
});
|
||||
|
||||
it('suppresses repeat notifications for the same author and assistant request', async () => {
|
||||
mocks.existing = true;
|
||||
|
||||
expect(await submitMechanicGapReport('author-1', parseMechanicGapReport(report)))
|
||||
.toEqual({ alreadySubmitted: true, recipientCount: 0 });
|
||||
expect(mocks.notifications).toHaveLength(0);
|
||||
expect(mocks.audits).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
import {
|
||||
AI_MECHANIC_SOURCE_PROFILES,
|
||||
AI_MECHANIC_SOURCE_KINDS,
|
||||
type AiMechanicCapabilityGap
|
||||
} from '$lib/ai/mechanic-authoring';
|
||||
import type { AiMechanicDeveloperHandoff as DeveloperHandoff } from '$lib/ai/mechanic-developer-handoff';
|
||||
import { recordIsacAudit } from '$lib/server/audit';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import { createNotification } from '$lib/server/notifications';
|
||||
|
||||
const GAP_AREAS = new Set<AiMechanicCapabilityGap['area']>([
|
||||
'event',
|
||||
'predicate',
|
||||
'reducer',
|
||||
'mutation',
|
||||
'outcome',
|
||||
'scope',
|
||||
'value_reference',
|
||||
'source_variant',
|
||||
'runtime'
|
||||
]);
|
||||
|
||||
export class MechanicGapReportError extends Error {
|
||||
constructor(message: string, readonly status = 400) {
|
||||
super(message);
|
||||
this.name = 'MechanicGapReportError';
|
||||
}
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new MechanicGapReportError(`${label} must be an object.`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function text(value: unknown, label: string, maxLength: number, allowEmpty = false) {
|
||||
if (typeof value !== 'string') throw new MechanicGapReportError(`${label} must be text.`);
|
||||
const result = value.trim();
|
||||
if (!allowEmpty && !result) throw new MechanicGapReportError(`${label} is required.`);
|
||||
if (result.length > maxLength) throw new MechanicGapReportError(`${label} is too long.`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function stringList(value: unknown, label: string, maxItems: number) {
|
||||
if (!Array.isArray(value) || value.length > maxItems) {
|
||||
throw new MechanicGapReportError(`${label} must contain at most ${maxItems} items.`);
|
||||
}
|
||||
return value.map((item, index) => text(item, `${label} item ${index + 1}`, 2_000));
|
||||
}
|
||||
|
||||
export function parseMechanicGapReport(value: unknown): DeveloperHandoff {
|
||||
const input = record(value, 'Mechanic gap report');
|
||||
const contextInput = record(input.context, 'Context');
|
||||
const sourceKind = text(contextInput.sourceKind, 'Source type', 80);
|
||||
if (!AI_MECHANIC_SOURCE_KINDS.includes(sourceKind as (typeof AI_MECHANIC_SOURCE_KINDS)[number])) {
|
||||
throw new MechanicGapReportError('Source type is not supported.');
|
||||
}
|
||||
const sourceProfile = AI_MECHANIC_SOURCE_PROFILES[
|
||||
sourceKind as (typeof AI_MECHANIC_SOURCE_KINDS)[number]
|
||||
];
|
||||
const columnsInput = input.valueColumns;
|
||||
if (!Array.isArray(columnsInput) || !columnsInput.length || columnsInput.length > 8) {
|
||||
throw new MechanicGapReportError('Value columns must contain between one and eight items.');
|
||||
}
|
||||
const transcriptInput = input.transcript;
|
||||
if (!Array.isArray(transcriptInput) || transcriptInput.length > 20) {
|
||||
throw new MechanicGapReportError('Clarification history must contain at most 20 items.');
|
||||
}
|
||||
const responseInput = record(input.response, 'Capability gap response');
|
||||
if (responseInput.status !== 'capability_gap') {
|
||||
throw new MechanicGapReportError('Only capability-gap responses can be submitted.');
|
||||
}
|
||||
const gapsInput = responseInput.gaps;
|
||||
if (!Array.isArray(gapsInput) || !gapsInput.length || gapsInput.length > 12) {
|
||||
throw new MechanicGapReportError('Capability gaps must contain between one and 12 items.');
|
||||
}
|
||||
const metaInput = record(responseInput.meta, 'Response metadata');
|
||||
|
||||
const valueColumns = columnsInput.map((column, index) => {
|
||||
const item = record(column, `Value column ${index + 1}`);
|
||||
return {
|
||||
key: text(item.key, `Value column ${index + 1} key`, 100),
|
||||
variantKey: text(item.variantKey, `Value column ${index + 1} variant`, 100),
|
||||
label: text(item.label, `Value column ${index + 1} label`, 100)
|
||||
};
|
||||
});
|
||||
if (
|
||||
valueColumns.length !== sourceProfile.variantKeys.length
|
||||
|| valueColumns.some((column, index) => column.variantKey !== sourceProfile.variantKeys[index])
|
||||
) {
|
||||
throw new MechanicGapReportError('Value columns do not match the mechanic source.');
|
||||
}
|
||||
if (input.allowAssignedAbility !== sourceProfile.allowAssignedAbility) {
|
||||
throw new MechanicGapReportError('Assigned-ability scope does not match the mechanic source.');
|
||||
}
|
||||
if (input.allowProtocolUpgradeScaling !== sourceProfile.allowProtocolUpgradeScaling) {
|
||||
throw new MechanicGapReportError('Protocol upgrade scaling does not match the mechanic source.');
|
||||
}
|
||||
|
||||
return {
|
||||
context: {
|
||||
sourceKind: sourceKind as DeveloperHandoff['context']['sourceKind'],
|
||||
sourceName: text(contextInput.sourceName, 'Source name', 200, true),
|
||||
sourceDescription: text(contextInput.sourceDescription, 'Source wording', 12_000)
|
||||
},
|
||||
valueColumns,
|
||||
allowAssignedAbility: sourceProfile.allowAssignedAbility,
|
||||
allowProtocolUpgradeScaling: sourceProfile.allowProtocolUpgradeScaling,
|
||||
transcript: transcriptInput.map((entry, index) => {
|
||||
const item = record(entry, `Clarification ${index + 1}`);
|
||||
const answer = item.answer;
|
||||
if (!['string', 'number', 'boolean'].includes(typeof answer)) {
|
||||
throw new MechanicGapReportError(`Clarification ${index + 1} answer is invalid.`);
|
||||
}
|
||||
return {
|
||||
questionId: text(item.questionId, `Clarification ${index + 1} ID`, 120),
|
||||
question: text(item.question, `Clarification ${index + 1} question`, 1_000),
|
||||
answer: typeof answer === 'string'
|
||||
? text(answer, `Clarification ${index + 1} answer`, 2_000)
|
||||
: answer as number | boolean
|
||||
};
|
||||
}),
|
||||
response: {
|
||||
status: 'capability_gap',
|
||||
summary: text(responseInput.summary, 'Gap summary', 4_000),
|
||||
assumptions: stringList(responseInput.assumptions, 'Assumptions', 20),
|
||||
warnings: stringList(responseInput.warnings, 'Warnings', 20),
|
||||
gaps: gapsInput.map((gap, index) => {
|
||||
const item = record(gap, `Gap ${index + 1}`);
|
||||
const area = text(item.area, `Gap ${index + 1} area`, 80);
|
||||
if (!GAP_AREAS.has(area as AiMechanicCapabilityGap['area'])) {
|
||||
throw new MechanicGapReportError(`Gap ${index + 1} area is not supported.`);
|
||||
}
|
||||
return {
|
||||
area: area as AiMechanicCapabilityGap['area'],
|
||||
requiredBehavior: text(item.requiredBehavior, `Gap ${index + 1} required behavior`, 4_000),
|
||||
whyUnsupported: text(item.whyUnsupported, `Gap ${index + 1} unsupported reason`, 4_000),
|
||||
developerChange: text(item.developerChange, `Gap ${index + 1} developer change`, 4_000),
|
||||
workaround: item.workaround === null
|
||||
? null
|
||||
: text(item.workaround, `Gap ${index + 1} workaround`, 4_000)
|
||||
};
|
||||
}),
|
||||
meta: {
|
||||
requestId: text(metaInput.requestId, 'Assistant request ID', 255),
|
||||
model: text(metaInput.model, 'Assistant model', 120),
|
||||
inputTokens: Math.max(0, Number(metaInput.inputTokens) || 0),
|
||||
outputTokens: Math.max(0, Number(metaInput.outputTokens) || 0),
|
||||
totalTokens: Math.max(0, Number(metaInput.totalTokens) || 0)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function notificationBody(report: DeveloperHandoff) {
|
||||
const source = report.context.sourceName || 'Untitled mechanic';
|
||||
const changes = report.response.gaps.map((gap, index) =>
|
||||
`${index + 1}. ${gap.area}: ${gap.developerChange}`
|
||||
).join('\n');
|
||||
return `${source} (${report.context.sourceKind}) needs mechanic-language support.
|
||||
|
||||
${report.response.summary}
|
||||
|
||||
Developer changes:
|
||||
${changes}`;
|
||||
}
|
||||
|
||||
export async function submitMechanicGapReport(actorId: string, report: DeveloperHandoff) {
|
||||
const database = await getDatabase();
|
||||
const requestId = report.response.meta.requestId;
|
||||
const generationEntityId = `${report.context.sourceKind}:${requestId}`;
|
||||
const reportEntityId = `mechanic_gap:${requestId}`;
|
||||
const generated = await (await database.prepare(`SELECT id FROM audit_log
|
||||
WHERE actor_id=? AND entity_type='ai_assistance' AND action='mechanic_ai.generated'
|
||||
AND entity_id=? AND json_extract(after_json,'$.responseStatus')='capability_gap'
|
||||
LIMIT 1`)).get(actorId, generationEntityId);
|
||||
if (!generated) {
|
||||
throw new MechanicGapReportError('This mechanic gap is not linked to your assistant session.', 409);
|
||||
}
|
||||
const existing = await (await database.prepare(`SELECT id FROM audit_log
|
||||
WHERE actor_id=? AND entity_type='ai_assistance' AND action='mechanic_gap.submitted'
|
||||
AND entity_id=? LIMIT 1`)).get(actorId, reportEntityId);
|
||||
if (existing) return { alreadySubmitted: true, recipientCount: 0 };
|
||||
|
||||
const recipients = await (await database.prepare(`
|
||||
SELECT DISTINCT users.id
|
||||
FROM users
|
||||
LEFT JOIN deleted_accounts deleted ON deleted.user_id=users.id
|
||||
LEFT JOIN user_status status ON status.user_id=users.id
|
||||
LEFT JOIN user_roles ON user_roles.user_id=users.id
|
||||
LEFT JOIN role_permissions ON role_permissions.role_id=user_roles.role_id
|
||||
LEFT JOIN permissions role_permission ON role_permission.id=role_permissions.permission_id
|
||||
LEFT JOIN user_permissions ON user_permissions.user_id=users.id
|
||||
LEFT JOIN permissions direct_permission ON direct_permission.id=user_permissions.permission_id
|
||||
WHERE deleted.user_id IS NULL AND status.blocked_at IS NULL
|
||||
AND (status.banned_until IS NULL OR status.banned_until<=?)
|
||||
AND (users.role='admin' OR role_permission.name='system.admin'
|
||||
OR direct_permission.name='system.admin')
|
||||
`)).all(new Date().toISOString());
|
||||
if (!recipients.length) {
|
||||
throw new MechanicGapReportError('No active administrators are available to receive this report.', 503);
|
||||
}
|
||||
|
||||
const auditLink = `/isac/audit?category=ai_assistance&action=mechanic_gap.submitted&q=${encodeURIComponent(requestId)}`;
|
||||
const notificationIds: string[] = [];
|
||||
for (const recipient of recipients) {
|
||||
const notificationId = await createNotification({
|
||||
userId: String(recipient.id),
|
||||
category: 'isac',
|
||||
source: 'ISAC AUTHORING ASSIST',
|
||||
title: `Mechanic language gap: ${report.context.sourceName || report.context.sourceKind}`,
|
||||
body: notificationBody(report),
|
||||
link: auditLink,
|
||||
actions: [{ label: 'OPEN AUDIT EVIDENCE', href: auditLink }],
|
||||
mandatory: true
|
||||
});
|
||||
if (notificationId) notificationIds.push(notificationId);
|
||||
}
|
||||
await recordIsacAudit({
|
||||
actorId,
|
||||
category: 'ai_assistance',
|
||||
action: 'mechanic_gap.submitted',
|
||||
entityId: reportEntityId,
|
||||
after: {
|
||||
requestId,
|
||||
sourceKind: report.context.sourceKind,
|
||||
sourceName: report.context.sourceName,
|
||||
sourceDescription: report.context.sourceDescription,
|
||||
summary: report.response.summary,
|
||||
assumptions: report.response.assumptions,
|
||||
warnings: report.response.warnings,
|
||||
gaps: report.response.gaps,
|
||||
recipientCount: notificationIds.length
|
||||
}
|
||||
});
|
||||
return { alreadySubmitted: false, recipientCount: notificationIds.length };
|
||||
}
|
||||
Reference in New Issue
Block a user