feat: use tool calls for answer extraction

This commit is contained in:
Erik Kotsiuba
2026-08-10 09:56:55 +02:00
parent 2bba854326
commit 0145055445
7 changed files with 313 additions and 114 deletions
+111
View File
@@ -2,6 +2,9 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
collectExtractionBusinessIssues,
createExtractionContext,
extractionCandidateFromContent,
parseExtractionCandidate,
repairExtractionParams,
selectExtractionModel,
} from "../src/answer-extraction.ts";
@@ -124,3 +127,111 @@ test("selectExtractionModel validates fallback model auth", async () => {
error: "No auth for fallback chat model: fallback/c.",
});
});
test("extraction context includes the tool and preceding turn", () => {
const context = createExtractionContext({
assistantText: "Which direction: A or B?",
attempt: 0,
lastError: "",
lastResponse: "",
previousUserText: "Choose an implementation direction.",
});
assert.equal(context.tools?.[0]?.name, "ask_user");
assert.equal(context.tools?.length, 1);
const prompt = JSON.stringify(context.messages[0]?.content);
assert.equal(prompt.includes("Choose an implementation direction."), true);
assert.equal(prompt.includes("Which direction: A or B?"), true);
});
test("tool-call arguments become the extraction candidate", () => {
const candidate = extractionCandidateFromContent([
{ type: "thinking", thinking: "ignored" },
{
type: "toolCall",
id: "call-1",
name: "ask_user",
arguments: {
title: "Direction",
questions: [
{
id: "direction",
prompt: "Which direction?",
options: [{ value: "a", label: "A" }],
},
],
},
},
] as never);
const result = parseExtractionCandidate(candidate);
assert.equal(result.ok, true);
if (result.ok) {
assert.equal(result.params.title, "Direction");
}
});
test("tool-call arguments take precedence over text", () => {
const candidate = extractionCandidateFromContent([
{ type: "text", text: "not JSON" },
{
type: "toolCall",
id: "call-1",
name: "ask_user",
arguments: { questions: [] },
},
] as never);
assert.equal(parseExtractionCandidate(candidate).ok, true);
});
test("multiple tool calls are rejected", () => {
const candidate = extractionCandidateFromContent([
{
type: "toolCall",
id: "call-1",
name: "ask_user",
arguments: { questions: [] },
},
{
type: "toolCall",
id: "call-2",
name: "ask_user",
arguments: { questions: [] },
},
] as never);
assert.equal(parseExtractionCandidate(candidate).ok, false);
});
test("fenced text remains a fallback", () => {
const result = parseExtractionCandidate('```json\n{"questions":[]}\n```');
assert.equal(result.ok, true);
});
test("semantic validation issues are returned for retry feedback", () => {
const result = parseExtractionCandidate(
JSON.stringify({
questions: [
{
id: "duplicate",
prompt: "First?",
options: [{ value: "a", label: "A" }],
},
{
id: "duplicate",
prompt: "Second?",
options: [{ value: "b", label: "B" }],
},
],
})
);
assert.equal(result.ok, true);
if (result.ok) {
assert.equal(
result.issues.some((issue) => issue.includes("duplicate")),
true
);
}
});