Let users choose the metadata generation model (#3215)

Expose Automatic and Manual metadata model selection in Host Settings, preserve existing configured fallbacks, and keep structured generation resilient through discovered candidates. Includes Playwright coverage, translations, public documentation, and settings-row alignment polish.
2026-08-11 14:01:00 +02:00
parent b599d38a77
commit 36a13601dd
19 changed files with 466 additions and 66 deletions
@@ -0,0 +1,110 @@
+import { expect, test, type Page } from "../support/fixtures";
+import { gotoAppShell, openSettings } from "../support/helpers/app";
+import { getServerId } from "../support/helpers/server-id";
+import { connectNewWorkspaceDaemonClient } from "../support/helpers/new-workspace";
+import {
+  expectSettingsHeader,
+  openHostSection,
+  openSettingsHost,
+} from "../support/helpers/settings";
+
+async function openMetadataGenerationSettings(page: Page) {
+  const serverId = getServerId();
+  await gotoAppShell(page);
+  await openSettings(page);
+  await openSettingsHost(page, serverId);
+  await openHostSection(page, serverId, "metadata");
+  await expectSettingsHeader(page, "Metadata");
+}
+
+async function openManualMetadataModelPicker(page: Page) {
+  await page.getByRole("button", { name: "Manual", exact: true }).click();
+  await page.getByRole("button", { name: /Select model/ }).click();
+}
+
+test("chooses a metadata model and can return to automatic selection", async ({
+  page,
+}, testInfo) => {
+  await openMetadataGenerationSettings(page);
+
+  await expect(page.getByText("Workspace titles", { exact: false })).toBeVisible();
+  await expect(page.getByRole("button", { name: "Automatic", exact: true })).toHaveAttribute(
+    "aria-selected",
+    "true",
+  );
+  await page.screenshot({
+    path: testInfo.outputPath("metadata-automatic.png"),
+    fullPage: true,
+  });
+
+  await openManualMetadataModelPicker(page);
+  await page.getByText("Mock Load Test", { exact: true }).click();
+  await expect(page.getByText("Ten second stream", { exact: true })).toBeVisible();
+  await page.screenshot({
+    path: testInfo.outputPath("metadata-model-picker.png"),
+    fullPage: true,
+  });
+  await page.getByText("Ten second stream", { exact: true }).click();
+
+  await expect(page.getByRole("button", { name: /Ten second stream/ })).toBeVisible();
+  await page.reload();
+  await expect(page.getByRole("button", { name: "Manual", exact: true })).toHaveAttribute(
+    "aria-selected",
+    "true",
+  );
+  await expect(page.getByRole("button", { name: /Ten second stream/ })).toBeVisible();
+  await page.screenshot({
+    path: testInfo.outputPath("metadata-manual-persisted.png"),
+    fullPage: true,
+  });
+
+  await page.getByRole("button", { name: "Automatic", exact: true }).click();
+  await page.reload();
+
+  await expect(page.getByRole("button", { name: "Automatic", exact: true })).toHaveAttribute(
+    "aria-selected",
+    "true",
+  );
+  await expect(page.getByRole("button", { name: /Ten second stream/ })).toHaveCount(0);
+});
+
+test("replaces only the first configured metadata model", async ({ page }) => {
+  const client = await connectNewWorkspaceDaemonClient({ ownProjects: false });
+  const previousConfig = await client.getDaemonConfig();
+
+  try {
+    await client.patchDaemonConfig({
+      metadataGeneration: {
+        providers: [
+          { provider: "mock", model: "five-minute-stream" },
+          { provider: "mock", model: "thirty-minute-stream" },
+        ],
+      },
+    });
+    await openMetadataGenerationSettings(page);
+
+    await expect(page.getByRole("button", { name: "Manual", exact: true })).toHaveAttribute(
+      "aria-selected",
+      "true",
+    );
+    await openManualMetadataModelPicker(page);
+    await page.getByText("Ten second stream", { exact: true }).click();
+
+    await expect
+      .poll(async () => (await client.getDaemonConfig()).config.metadataGeneration.providers)
+      .toEqual([
+        { provider: "mock", model: "ten-second-stream" },
+        { provider: "mock", model: "thirty-minute-stream" },
+      ]);
+  } finally {
+    try {
+      await client.patchDaemonConfig({
+        metadataGeneration: {
+          providers: previousConfig.config.metadataGeneration.providers,
+        },
+      });
+    } finally {
+      await client.close().catch(() => undefined);
+    }
+  }
+});
@@ -36,6 +36,7 @@ type HostSection =
  | "connections"
  | "pair-device"
  | "agents"
+  | "metadata"
  | "workspaces"
  | "providers"
  | "usage"