jb/pi-ask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add answer extraction replay commands

Browse Source
This commit is contained in:
Erik Kotsiuba
2026-05-02 22:47:55 +02:00
parent 288caf49de
commit 5b70eca306
36 changed files with 1636 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/answer-extraction.test.ts
+126
View File
@@ -0,0 +1,126 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
collectExtractionBusinessIssues,
repairExtractionParams,
selectExtractionModel,
} from "../src/answer-extraction.ts";
function model(provider: string, id: string) {
return { provider, id } as never;
}
test("selectExtractionModel uses first configured model with auth", async () => {
const first = model("missing", "a");
const second = model("ok", "b");
const result = await selectExtractionModel(
{
model: model("fallback", "c"),
modelRegistry: {
find(provider: string, _id: string) {
if (provider === "missing") {
return first;
}
if (provider === "ok") {
return second;
}
},
getApiKeyAndHeaders(candidate: { provider: string }) {
if (candidate.provider === "ok") {
return Promise.resolve({ ok: true, apiKey: "key" });
}
return Promise.resolve({ ok: false, error: "no" });
},
},
} as never,
[
{ provider: "missing", id: "a" },
{ provider: "ok", id: "b" },
]
);
assert.deepEqual(result, {
model: second,
auth: { ok: true, apiKey: "key" },
usedFallback: false,
});
});
test("extraction business rules flag option spam", () => {
const issues = collectExtractionBusinessIssues({
questions: [
{
id: "q",
prompt: "Pick one",
options: [
{ value: "a", label: "A" },
{ value: "b", label: "B" },
{ value: "c", label: "C" },
{ value: "d", label: "D" },
{ value: "e", label: "E" },
],
},
],
});
assert.deepEqual(issues, ["questions[0].options has 5 items; max is 4"]);
});
test("repairExtractionParams caps options while preserving other", () => {
const repaired = repairExtractionParams({
questions: [
{
id: "q",
prompt: "Pick one",
options: [
{ value: "a", label: "A" },
{ value: "b", label: "B" },
{ value: "c", label: "C" },
{ value: "d", label: "D" },
{ value: "other", label: "Other" },
],
},
],
});
assert.deepEqual(
repaired.questions[0]?.options.map((option) => option.value),
["a", "b", "c", "other"]
);
});
test("repairExtractionParams drops generic conversational prompts", () => {
const repaired = repairExtractionParams({
questions: [
{
id: "generic",
prompt: "How can I help you today?",
options: [{ value: "help", label: "Help me" }],
},
],
});
assert.deepEqual(repaired.questions, []);
});
test("selectExtractionModel validates fallback model auth", async () => {
const fallback = model("fallback", "c");
const result = await selectExtractionModel(
{
model: fallback,
modelRegistry: {
find() {
return;
},
getApiKeyAndHeaders() {
return Promise.resolve({ ok: false, error: "no auth" });
},
},
} as never,
[]
);
assert.deepEqual(result, {
error: "No auth for fallback chat model: fallback/c.",
});
});
tests/ask-payload-store.test.ts
+95
View File
@@ -0,0 +1,95 @@
import assert from "node:assert/strict";
import test from "node:test";
import { findLatestPayloadInCurrentBranch } from "../src/ask-payload-store.ts";
import type { AskParams } from "../src/types.ts";
const params: AskParams = {
questions: [
{
id: "goal",
prompt: "Goal?",
options: [{ value: "a", label: "A" }],
},
],
};
function custom(data: unknown) {
return { type: "custom", customType: "ask:payload", data };
}
function ctx(branch: unknown[]) {
return { sessionManager: { getBranch: () => branch } } as never;
}
test("branch payload lookup returns the latest valid matching source", () => {
const older = {
version: 1,
source: "tool",
params,
timestamp: 1,
};
const newer = {
version: 1,
source: "tool",
params: { ...params, title: "Newer" },
timestamp: 2,
};
const result = findLatestPayloadInCurrentBranch(
ctx([custom(older), custom(newer)]),
"tool"
);
assert.equal(result.data, newer);
assert.equal(result.invalidMatchFound, false);
});
test("branch payload lookup allows freeform only for answer extraction payloads", () => {
const freeformParams: AskParams = {
questions: [
{
id: "language",
prompt: "Which language?",
options: [{ value: "freeform", label: "Type answer", freeform: true }],
},
],
};
const answerPayload = {
version: 1,
source: "answer-extraction",
params: freeformParams,
timestamp: 1,
};
const toolPayload = {
version: 1,
source: "tool",
params: freeformParams,
timestamp: 2,
};
assert.equal(
findLatestPayloadInCurrentBranch(
ctx([custom(answerPayload)]),
"answer-extraction"
).data,
answerPayload
);
assert.equal(
findLatestPayloadInCurrentBranch(ctx([custom(toolPayload)]), "tool").data,
undefined
);
});
test("branch payload lookup ignores invalid stored payloads", () => {
const result = findLatestPayloadInCurrentBranch(
ctx([
custom({ version: 1, source: "tool", params: { questions: [] } }),
custom({ version: 1, source: "answer-extraction", params }),
]),
"tool"
);
assert.equal(result.data, undefined);
assert.equal(result.invalidMatchFound, true);
});
tests/ask-tool.test.ts
+40 -16
View File
@@ -24,19 +24,26 @@ const noop = () => {
function registerMockTool() {
const tools: Record<string, unknown>[] = [];
const entries: Array<{ customType: string; data: unknown }> = [];
registerAskTool({
appendEntry(customType: string, data: unknown) {
entries.push({ customType, data });
},
registerTool(tool: unknown) {
tools.push(tool as Record<string, unknown>);
},
} as never);
return tools[0] as {
execute: (...args: any[]) => Promise<any>;
renderCall: (args: unknown, theme: any) => { text: string };
renderResult: (
result: any,
options: unknown,
theme: any
) => { text: string };
return {
entries,
tool: tools[0] as {
execute: (...args: any[]) => Promise<any>;
renderCall: (args: unknown, theme: any) => { text: string };
renderResult: (
result: any,
options: unknown,
theme: any
) => { text: string };
},
};
}
@@ -61,8 +68,25 @@ function sampleParams(): AskParams {
};
}
test("ask tool stores valid payloads as soon as they are called", async () => {
const { entries, tool } = registerMockTool();
const params = sampleParams();
await tool.execute("call-1", params, undefined, noop, makeCtx(false));
assert.equal(entries.length, 1);
assert.equal(entries[0].customType, "ask:payload");
assert.deepEqual(entries[0].data, {
version: 1,
source: "tool",
params,
sourceEntryId: "call-1",
timestamp: (entries[0].data as { timestamp: number }).timestamp,
});
});
test("ask tool returns pending questions in non-interactive mode", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -90,7 +114,7 @@ test("ask tool returns pending questions in non-interactive mode", async () => {
});
test("ask tool includes custom answer fallback for preview questions", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -115,7 +139,7 @@ test("ask tool includes custom answer fallback for preview questions", async ()
});
test("ask tool rejects invalid payloads before UI opens with structured issues", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -155,7 +179,7 @@ test("ask tool rejects invalid payloads before UI opens with structured issues",
});
test("ask tool reports missing option values with structured issues", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -188,7 +212,7 @@ test("ask tool reports missing option values with structured issues", async () =
});
test("ask tool reports empty questions with structured issues", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -215,7 +239,7 @@ test("ask tool reports empty questions with structured issues", async () => {
});
test("ask tool reports invalid question types with structured issues", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -250,7 +274,7 @@ test("ask tool reports invalid question types with structured issues", async ()
});
test("ask tool reports preview validation with structured issues", async () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const result = await tool.execute(
"call-1",
@@ -285,7 +309,7 @@ test("ask tool reports preview validation with structured issues", async () => {
});
test("ask tool transcript renderers summarize call and cancelled result", () => {
const tool = registerMockTool();
const { tool } = registerMockTool();
const theme = {
bold: (text: string) => text,
fg: (_token: string, text: string) => text,
tests/config-store.test.ts
+6 -3
View File
@@ -24,7 +24,8 @@ test("config store writes defaults when file is missing", async () => {
assert.deepEqual(result.config, DEFAULT_ASK_CONFIG);
assert.deepEqual(JSON.parse(await readFile(path, "utf-8")), {
schemaVersion: 1,
schemaVersion: 2,
answer: DEFAULT_ASK_CONFIG.answer,
behaviour: DEFAULT_ASK_CONFIG.behaviour,
keymaps: DEFAULT_ASK_CONFIG.keymaps,
});
@@ -47,7 +48,8 @@ test("config store writes full normalized config on save", async () => {
const content = await readFile(path, "utf-8");
assert.deepEqual(JSON.parse(content), {
schemaVersion: 1,
schemaVersion: 2,
answer: DEFAULT_ASK_CONFIG.answer,
behaviour: {
autoSubmitWhenAnsweredWithoutNotes: true,
confirmDismissWhenDirty: true,
@@ -143,7 +145,8 @@ test("config store migrates the legacy root config path into extensions", async
assert.equal(result.config.behaviour.showFooterHints, false);
await assert.rejects(readFile(legacyPath, "utf-8"));
assert.deepEqual(JSON.parse(await readFile(path, "utf-8")), {
schemaVersion: 1,
schemaVersion: 2,
answer: DEFAULT_ASK_CONFIG.answer,
behaviour: {
autoSubmitWhenAnsweredWithoutNotes: true,
confirmDismissWhenDirty: true,
tests/editor-state.test.ts
+85
View File
@@ -66,6 +66,91 @@ test("submitEditorDraft advances submitted custom answers", () => {
assert.equal(next.view.kind, "submit");
});
test("submitted answer can sync directly into next freeform input", () => {
let state = syncStateToSelection(
createInitialState(
{
questions: [
{
id: "q1",
prompt: "First?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
{
id: "q2",
prompt: "Second?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
},
{ allowFreeform: true }
)
);
state = submitEditorDraft(state, "first answer");
state = syncStateToSelection(state);
assert.equal(state.activeTabIndex, 1);
assert.deepEqual(state.view, { kind: "input", questionId: "q2" });
});
test("syncStateToSelection does not reopen answered freeform input", () => {
let state = syncStateToSelection(
createInitialState(
{
questions: [
{
id: "q1",
prompt: "First?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
},
{ allowFreeform: true }
)
);
state = submitEditorDraft(state, "already answered");
state = syncStateToSelection({
...state,
activeTabIndex: 0,
view: { kind: "navigate" },
});
assert.equal(state.view.kind, "navigate");
assert.equal(state.answers.q1.customText, "already answered");
});
test("tab navigation can sync from review back into freeform input", () => {
let state = createInitialState(
{
questions: [
{
id: "q1",
prompt: "First?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
},
{ allowFreeform: true }
);
state = { ...state, activeTabIndex: 1, view: { kind: "submit" } };
state = syncStateToSelection({
...state,
activeTabIndex: 0,
view: { kind: "navigate" },
});
assert.deepEqual(state.view, { kind: "input", questionId: "q1" });
});
test("submitEditorDraft saves notes without completing the ask flow", () => {
const state = enterQuestionNoteMode(
createInitialState({
tests/render-question.test.ts
+41
View File
@@ -189,6 +189,47 @@ test("selected multiline custom option renders full editor block", () => {
assert(!lines.some((line) => line.includes("first line second line")));
});
test("freeform-only question renders label without numbering or pointer and separates input", () => {
const state = createInitialState(
{
questions: [
{
id: "q1",
prompt: "Type one",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
},
{ allowFreeform: true }
);
const lines: string[] = [];
renderQuestionScreen({
editor: mockEditor("", ["┌────┐", "", "└────┘"]),
lines,
options: getRenderableOptions(state.questions[0]),
question: state.questions[0],
state: { ...state, view: { kind: "input", questionId: "q1" } },
theme: mockTheme(),
width: 80,
});
const labelIndex = lines.findIndex((line) =>
line.includes("Type your answer:")
);
const inputIndex = lines.findIndex((line) => line.includes("Type answer..."));
assert.notEqual(labelIndex, -1);
assert.equal(inputIndex, labelIndex + 2);
assert(lines[labelIndex]?.startsWith(" "));
assert(lines[inputIndex]?.startsWith(" "));
assert(!lines[inputIndex]?.startsWith(" "));
assert(!lines.some((line) => line.includes("1. Type your answer:")));
assert(!lines.some((line) => line.includes("❯")));
});
test("preview questions also show the custom answer option", () => {
const state = createInitialState({
questions: [
tests/settings-list.test.ts
+4
View File
@@ -1,10 +1,14 @@
import assert from "node:assert/strict";
import test from "node:test";
import { visibleWidth } from "@mariozechner/pi-tui";
import { DEFAULT_ASK_CONFIG } from "../src/config/defaults.ts";
import type { AskConfig } from "../src/config/schema.ts";
import { AskSettingsList } from "../src/ui/settings-list.ts";
const savedConfig: AskConfig = {
answer: {
...DEFAULT_ASK_CONFIG.answer,
},
behaviour: {
autoSubmitWhenAnsweredWithoutNotes: false,
confirmDismissWhenDirty: true,
tests/state.test.ts
+65
View File
@@ -6,6 +6,9 @@ const DUPLICATE_QUESTION_ID_RE = /duplicate question id "scope"/;
const DUPLICATE_OPTION_VALUE_RE = /duplicate option value "small"/;
const PREVIEW_REQUIRED_RE =
/preview questions require preview text for every option/;
const FREEFORM_UNSUPPORTED_RE =
/freeform options are only supported for \/answer forms/;
const FREEFORM_MIXED_RE = /freeform options must be the only option/;
import { toAskResult } from "../src/state/result.ts";
import { getRenderableOptions } from "../src/state/selectors.ts";
@@ -76,6 +79,68 @@ test("normalize defaults via initial state", () => {
);
});
test("answer freeform questions render only the custom input option", () => {
const state = createInitialState(
{
questions: [
{
id: "language",
prompt: "Which language do you want to learn?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
},
{ allowFreeform: true }
);
const options = getRenderableOptions(state.questions[0]);
assert.equal(options.length, 1);
assert.equal(options[0]?.label, "Type your answer:");
assert.equal(options[0]?.isCustomOption, true);
});
test("freeform questions are rejected unless explicitly allowed", () => {
assert.throws(
() =>
createInitialState({
questions: [
{
id: "language",
prompt: "Which language do you want to learn?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
],
},
],
}),
FREEFORM_UNSUPPORTED_RE
);
});
test("freeform questions cannot mix normal options", () => {
assert.throws(
() =>
createInitialState(
{
questions: [
{
id: "language",
prompt: "Which language do you want to learn?",
options: [
{ value: "freeform", label: "Type answer", freeform: true },
{ value: "french", label: "French" },
],
},
],
},
{ allowFreeform: true }
),
FREEFORM_MIXED_RE
);
});
test("preview questions keep their type and option previews", () => {
const state = createInitialState({
questions: [