Compare commits
81
Commits
59374b538b
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b667b9ffc4 | ||
|
|
0cc0e82bbc | ||
|
|
284fdd638d | ||
|
|
9862ef0be9 | ||
|
|
5ddd98bf5f | ||
|
|
54303e0b5d | ||
|
|
2dc359c48d | ||
|
|
fe643e2d8d | ||
|
|
12a6bb6afb | ||
|
|
6a725c1f47 | ||
|
|
f2e42f5faf | ||
|
|
acd57ae2a7 | ||
|
|
e10a9573b9 | ||
|
|
574e083e75 | ||
|
|
4360f01947 | ||
|
|
c0c18a5191 | ||
|
|
3c321b3033 | ||
|
|
a79418efcb | ||
|
|
05793fe7b0 | ||
|
|
6fba62800a | ||
|
|
773c628512 | ||
|
|
89474017e1 | ||
|
|
fd63b0ccec | ||
|
|
fd0edb1511 | ||
|
|
8d3552bc92 | ||
|
|
45d6a41a39 | ||
|
|
eb52030a38 | ||
|
|
ea05b2f2a7 | ||
|
|
5b1e771ec1 | ||
|
|
fee0d44e2d | ||
|
|
e5a2e53a94 | ||
|
|
42dd6efdf3 | ||
|
|
e1c5c55c15 | ||
|
|
3cf354b74b | ||
|
|
328faa03e0 | ||
|
|
3d68f7c6bf | ||
|
|
bda414abbe | ||
|
|
f61f197dbc | ||
|
|
070ba3dafa | ||
|
|
9d68d5f81b | ||
|
|
5f853dcec0 | ||
|
|
7e7cfdb5cb | ||
|
|
88c20ca51b | ||
|
|
574adac57f | ||
|
|
ebbf3c3d19 | ||
|
|
c257f31c21 | ||
|
|
435a12107f | ||
|
|
26eeab209e | ||
|
|
4178ee4857 | ||
|
|
52576c7556 | ||
|
|
444483ade6 | ||
|
|
ebab671969 | ||
|
|
b556bcf147 | ||
|
|
b4e71b47db | ||
|
|
1782d1a0c6 | ||
|
|
a241fb05ae | ||
|
|
7a58ec0eef | ||
|
|
08ebfb50ad | ||
|
|
e65ea4c0aa | ||
|
|
1e213db45a | ||
|
|
1bab01799c | ||
|
|
353278aa96 | ||
|
|
f8a4362287 | ||
|
|
12495473ee | ||
|
|
f1b525cec9 | ||
|
|
b6bb5db033 | ||
|
|
8c44c07c6a | ||
|
|
a794a1e10a | ||
|
|
6389d22f15 | ||
|
|
fbf1a52b64 | ||
|
|
06e631504c | ||
|
|
b4373113a4 | ||
|
|
27cbe965bb | ||
|
|
0a061ae44b | ||
|
|
7a1043e5e0 | ||
|
|
d99dbce93d | ||
|
|
0423ec3676 | ||
|
|
2c3ee1f014 | ||
|
|
67877297d0 | ||
|
|
a11ca1743c | ||
|
|
9e1d59fdee |
@@ -0,0 +1,81 @@
|
||||
name: fast-browser scenarios
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "extensions/fast-browser/**"
|
||||
- "package.json"
|
||||
- "bun.lock"
|
||||
- ".github/workflows/fast-browser-scenarios.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "extensions/fast-browser/**"
|
||||
- "package.json"
|
||||
- "bun.lock"
|
||||
- ".github/workflows/fast-browser-scenarios.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
offline:
|
||||
name: Offline scenario contract tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3.14"
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install pinned dependencies
|
||||
run: |
|
||||
bun install --frozen-lockfile
|
||||
python -m pip install --disable-pip-version-check 'browser-harness==0.1.13'
|
||||
- name: Run offline scenario contract tests
|
||||
run: |
|
||||
node --test --experimental-strip-types \
|
||||
extensions/fast-browser/scenario-cli.test.ts \
|
||||
extensions/fast-browser/scenarios.test.ts \
|
||||
extensions/fast-browser/reports.test.ts
|
||||
python -m unittest discover -s extensions/fast-browser -p 'test_*.py'
|
||||
- name: Validate local scenario examples without a browser
|
||||
run: |
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --check --format json extensions/fast-browser/scenarios/examples/local-form.json
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --check --format json extensions/fast-browser/scenarios/examples/local-login.json
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --check --format json extensions/fast-browser/scenarios/examples/local-popup.json
|
||||
|
||||
live-headless:
|
||||
name: Local Chromium headless scenarios
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3.14"
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install pinned browser dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends chromium
|
||||
python -m venv .pi/fast-browser-demo/venv
|
||||
.pi/fast-browser-demo/venv/bin/python -m pip install --disable-pip-version-check 'browser-harness==0.1.13'
|
||||
bun install --frozen-lockfile
|
||||
- name: Run declarative CLI scenarios against local Chromium
|
||||
env:
|
||||
FAST_BROWSER_FIXTURE_MODE: "local"
|
||||
FAST_BROWSER_TEST_PASSWORD: "demo-password"
|
||||
FAST_BROWSER_PYTHON: "${{ github.workspace }}/.pi/fast-browser-demo/venv/bin/python"
|
||||
run: |
|
||||
report_dir="$RUNNER_TEMP/fast-browser-scenario-reports"
|
||||
mkdir -m 700 -p "$report_dir"
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --headless --capture-on-request --format json --report "$report_dir/local-form.json" extensions/fast-browser/scenarios/examples/local-form.json
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --headless --capture-on-request --format json --report "$report_dir/local-login.json" extensions/fast-browser/scenarios/examples/local-login.json
|
||||
node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --headless --capture-on-request --format json --report "$report_dir/local-popup.json" extensions/fast-browser/scenarios/examples/local-popup.json
|
||||
node --input-type=module -e 'import { readFileSync, accessSync } from "node:fs"; for (const name of ["local-form", "local-login", "local-popup"]) { const report = JSON.parse(readFileSync(`${process.env.RUNNER_TEMP}/fast-browser-scenario-reports/${name}.json`, "utf8")); for (const evidence of report.evidence ?? []) if (evidence.status === "captured") accessSync(evidence.retainedPath); }'
|
||||
@@ -7,11 +7,13 @@ Personal [Pi](https://github.com/earendil-works/pi-mono) extensions, skills, glo
|
||||
[`config/pi-links.json`](config/pi-links.json) is the source of truth for the normal installation. The link manager exposes:
|
||||
|
||||
- global agent instructions for Pi, Codex, and Claude;
|
||||
- 34 shared skills under `~/.agents/skills/`;
|
||||
- 35 shared skills under `~/.agents/skills/`, including the model-invoked [`develop-workflow`](skills/develop/develop-workflow/SKILL.md) skill;
|
||||
- the `manage-pi-links` and `nono-profiles` project-only skills under `.pi/skills/`;
|
||||
- no global Pi extension symlinks. Active extensions are installed separately as pinned Gitea npm packages.
|
||||
|
||||
The repository contains `copy-all`, `dictate`, `effort-controls`, `file-search`, `interactive-subagents`, `lm-studio`, `native-hooks`, `prompt-snippets`, `subagents`, `todo-herdr`, `update`, and the private `workflow` extension. The link manifest does not install any of them globally. The `pptx`, `stop-slop`, `subagents`, and `xlsx` skills are also intentionally excluded. Archived resources under `__archived/` are never installed. Update `config/pi-links.json` deliberately if that policy changes.
|
||||
The repository contains `bash-cwd`, `copy-all`, `dictate` (Linux host-service dictation), `effort-controls`, `fast-browser` (experimental local JEV browser control), `file-search`, `interactive-subagents`, `lm-studio`, `native-hooks`, `prompt-snippets`, `subagents`, `todo-herdr`, and `update`. The link manifest does not install any extension globally. New feature work uses `develop-workflow`; both legacy workflow implementations are retained under [`__archived/extensions/`](__archived/extensions/README.md) per [ADR 0007](docs/adr/0007-archive-legacy-workflows-05SEP2026.md). The `pptx`, `stop-slop`, `subagents`, and `xlsx` skills are intentionally excluded. Archived resources under `__archived/` are never installed. Update `config/pi-links.json` deliberately if that policy changes.
|
||||
|
||||
The experimental [`fast-browser` package](extensions/fast-browser/README.md) runs caller-bounded local-browser tests with JEV candidate selection, optional Pi-model text generation, and local test credentials. It supports unattended automation, file/TUI/per-call settings, isolated Chromium-family browser sessions or explicit CDP attachment, and exclusively locked named profiles with manual sign-in setup. See the [feasibility review](docs/browser-use-jev-feasibility.md) and [prototype plan](docs/fast-browser-prototype-plan.md).
|
||||
|
||||
## Fresh-machine setup
|
||||
|
||||
@@ -91,13 +93,13 @@ Use `bun install --frozen-lockfile` after pulling a lockfile change and plain `b
|
||||
Preview the exact changes first, then apply them:
|
||||
|
||||
```bash
|
||||
cd /srv/data/code/harness-tools
|
||||
cd /srv/data/code/jb-pi/harness-tools
|
||||
./scripts/pi-links.mjs sync --dry-run
|
||||
./scripts/pi-links.mjs sync
|
||||
./scripts/pi-links.mjs check
|
||||
```
|
||||
|
||||
A successful check currently reports 41 links and zero local package entries. The manager:
|
||||
A successful check currently reports 39 links and zero local package entries. The manager:
|
||||
|
||||
- preserves unrelated global resources;
|
||||
- refuses to overwrite regular files, directories, or links owned outside this repository;
|
||||
@@ -126,12 +128,14 @@ Inside Pi:
|
||||
|
||||
The currently published and deployed extension set is:
|
||||
|
||||
- `@jbcode/pi-dictate@0.1.0`;
|
||||
- `@jbcode/pi-dictate@0.2.0`;
|
||||
- `@jbcode/pi-effort-controls@0.1.0`;
|
||||
- `@jbcode/pi-prompt-snippets@0.1.0`;
|
||||
- `@jbcode/pi-interactive-subagents@3.7.3`;
|
||||
- `@jbcode/pi-todo-herdr@0.2.5`.
|
||||
|
||||
[`@jbcode/pi-bash-cwd`](extensions/bash-cwd) is an unpublished compatibility package for Pi 0.84.x. It adds a validated optional `cwd` argument to the built-in `bash` tool so commands can target hidden worktrees and nested packages without an inline `cd`. Keep using explicit directory proofs for Git worktree mutations even with the override installed.
|
||||
|
||||
Configure the scoped Gitea npm registry and install only the packages you want. Keep the token in the user environment or user-level npm configuration; never commit it:
|
||||
|
||||
```ini
|
||||
@@ -140,7 +144,8 @@ Configure the scoped Gitea npm registry and install only the packages you want.
|
||||
```
|
||||
|
||||
```bash
|
||||
pi install npm:@jbcode/pi-dictate@0.1.0
|
||||
pi install npm:@jbcode/pi-fast-browser@0.1.0
|
||||
pi install npm:@jbcode/pi-dictate@0.2.0
|
||||
pi install npm:@jbcode/pi-effort-controls@0.1.0
|
||||
pi install npm:@jbcode/pi-prompt-snippets@0.1.0
|
||||
pi install npm:@jbcode/pi-interactive-subagents@3.7.3
|
||||
@@ -435,38 +440,43 @@ Do not place real provider keys in files readable by sandboxed Pi if the goal is
|
||||
|
||||
## Optional Dictate service
|
||||
|
||||
Dictate is Linux-only and requires:
|
||||
Dictate is a Linux-only host service. Every backend requires:
|
||||
|
||||
- PipeWire with `pw-record`;
|
||||
- systemd user services;
|
||||
- an NVIDIA GPU with a working CUDA runtime;
|
||||
- `uv`;
|
||||
- network access for the initial locked dependency and model download;
|
||||
- UID 1000 unless the hardcoded socket path is changed first.
|
||||
- access to the exact `/run/user/1000/pi-dictate.sock`, unless the socket path is changed for another UID.
|
||||
|
||||
Install the host service from the repository root:
|
||||
The default backend is local NeMo/Parakeet. Local installation additionally requires an NVIDIA GPU, a working CUDA runtime, network access for initial provisioning, and several gigabytes for the locked environment and pinned model. Meta and OpenAI are explicit cloud opt-ins. Cloud-only installation does not install NeMo, CUDA, or model weights, but it requires a dedicated host credential and outbound DNS/TLS access from the service.
|
||||
|
||||
Install one backend from the repository root:
|
||||
|
||||
```bash
|
||||
cd /srv/data/code/harness-tools
|
||||
./services/dictate/install.sh
|
||||
./services/dictate/install.sh install --backend local
|
||||
# or:
|
||||
./services/dictate/install.sh install --backend meta --api-key-file /path/to/meta-key
|
||||
./services/dictate/install.sh install --backend openai --api-key-file /path/to/openai-key
|
||||
```
|
||||
|
||||
`--api-key-file` copies a regular source file into the host-owned mode-`0600` credential directory. Never pass a literal key. Configure an installed backend or validate it with:
|
||||
|
||||
```bash
|
||||
./services/dictate/install.sh configure --backend local
|
||||
./services/dictate/install.sh check
|
||||
systemctl --user status pi-dictate.socket pi-dictate.service
|
||||
```
|
||||
|
||||
The installer provisions the locked Python environment, verifies the pinned model, installs host-owned systemd units, and enables socket activation. Initial storage is several gigabytes. To reuse an existing model archive or preview the operation:
|
||||
Local mode remains offline after provisioning and uses the checksum-pinned Parakeet archive. Cloud mode streams bounded PCM to the selected provider only after its session is authenticated and ready. Backend changes are privileged host operations, require validation and a service restart, and fail closed if systemd network hardening does not match the selection. The service does not write audio files or expose provider credentials to Pi. Cloud provider retention, terms, and billing policies apply to audio and configured context sent with a cloud session.
|
||||
|
||||
```bash
|
||||
./services/dictate/install.sh --model-source /path/to/parakeet-tdt-0.6b-v3.nemo
|
||||
./services/dictate/install.sh --dry-run --model-source /path/to/model.nemo
|
||||
```
|
||||
|
||||
Restart a sandboxed Pi after profile or socket changes. In Pi, press `Alt+M` to start or stop dictation and `Alt+N` to cancel. See [`extensions/dictate/README.md`](extensions/dictate/README.md) for privacy, dictionary, model, and troubleshooting details.
|
||||
Restart a sandboxed Pi after profile or socket changes. In Pi, press `Alt+M` to start or stop dictation, `Alt+N` or `/dictate-cancel` to cancel, and `/dictate-status` to display bounded backend/model diagnostics. See [`extensions/dictate/README.md`](extensions/dictate/README.md) for configuration, provider links, privacy, dictionary, model, and troubleshooting details.
|
||||
|
||||
## Maintenance
|
||||
|
||||
After pulling repository changes:
|
||||
|
||||
```bash
|
||||
cd /srv/data/code/harness-tools
|
||||
cd /srv/data/code/jb-pi/harness-tools
|
||||
git pull --rebase
|
||||
bun install --frozen-lockfile
|
||||
./scripts/pi-links.mjs sync --dry-run
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# Archived extensions
|
||||
|
||||
`workflow/` and `_workflows/` are retired historical implementations, archived by explicit approval under [ADR 0007](../../docs/adr/0007-archive-legacy-workflows-05SEP2026.md). New development uses the [`develop-workflow` skill](../../skills/develop/develop-workflow/SKILL.md).
|
||||
|
||||
Their source and tests are retained for reference, excluded from active workspaces, typechecking, and root test commands. Historical source still contains relative imports into the former `extensions/shared` location and workspace dependency declarations; the archive is not a runnable or installable package set. To investigate the runnable historical layout, inspect commit `fee0d44e2ddd7a982e94ebf99117f69bb9bae78e` in a separate checkout rather than wiring archived files into the active runtime.
|
||||
|
||||
Historical plans and ADRs retain their original paths as evidence. The separate scenario and real-change pilot validation TODO remains open.
|
||||
@@ -0,0 +1,5 @@
|
||||
# workflow
|
||||
|
||||
> **Historical reference:** This private legacy extension was superseded for new development by the model-invoked [`develop-workflow` skill](../../../skills/develop/develop-workflow/SKILL.md) under [ADR 0006](../../../docs/adr/0006-skill-led-development-workflow-04SEP2026.md).
|
||||
|
||||
Keep this workspace private, unpublished, and unexposed during the skill pilot. It is historical implementation evidence, not the skill implementation or a prerequisite for skill development. Archive or remove it only in the separately approved post-pilot cleanup.
|
||||
@@ -18,6 +18,19 @@
|
||||
"vitest": "catalog:",
|
||||
},
|
||||
},
|
||||
"extensions/bash-cwd": {
|
||||
"name": "@jbcode/pi-bash-cwd",
|
||||
"version": "0.1.0",
|
||||
"devDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "catalog:",
|
||||
"typebox": "catalog:",
|
||||
"typescript": "catalog:",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
"typebox": "catalog:pi-peer",
|
||||
},
|
||||
},
|
||||
"extensions/copy-all": {
|
||||
"name": "@jbcode/pi-copy-all",
|
||||
"version": "0.1.0",
|
||||
@@ -27,7 +40,7 @@
|
||||
},
|
||||
"extensions/dictate": {
|
||||
"name": "@jbcode/pi-dictate",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
},
|
||||
@@ -39,6 +52,16 @@
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
},
|
||||
},
|
||||
"extensions/fast-browser": {
|
||||
"name": "@jbcode/pi-fast-browser",
|
||||
"version": "0.1.1",
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-ai": "catalog:pi-peer",
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
"@earendil-works/pi-tui": "catalog:pi-peer",
|
||||
"typebox": "catalog:pi-peer",
|
||||
},
|
||||
},
|
||||
"extensions/file-search": {
|
||||
"name": "@jbcode/pi-file-search",
|
||||
"version": "0.1.0",
|
||||
@@ -56,7 +79,7 @@
|
||||
},
|
||||
"extensions/interactive-subagents": {
|
||||
"name": "@jbcode/pi-interactive-subagents",
|
||||
"version": "3.7.3",
|
||||
"version": "3.7.4",
|
||||
"devDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "catalog:",
|
||||
"@earendil-works/pi-tui": "catalog:",
|
||||
@@ -147,19 +170,6 @@
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
},
|
||||
},
|
||||
"extensions/workflow": {
|
||||
"name": "@jbcode/pi-workflow",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@jbcode/pi-shared": "workspace:^",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-ai": "catalog:pi-peer",
|
||||
"@earendil-works/pi-coding-agent": "catalog:pi-peer",
|
||||
"@earendil-works/pi-tui": "catalog:pi-peer",
|
||||
"typebox": "catalog:pi-peer",
|
||||
},
|
||||
},
|
||||
},
|
||||
"overrides": {
|
||||
"@effect/platform-node-shared": "4.0.0-beta.101",
|
||||
@@ -354,12 +364,16 @@
|
||||
|
||||
"@ioredis/commands": ["@ioredis/commands@1.10.0", "", {}, "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q=="],
|
||||
|
||||
"@jbcode/pi-bash-cwd": ["@jbcode/pi-bash-cwd@workspace:extensions/bash-cwd"],
|
||||
|
||||
"@jbcode/pi-copy-all": ["@jbcode/pi-copy-all@workspace:extensions/copy-all"],
|
||||
|
||||
"@jbcode/pi-dictate": ["@jbcode/pi-dictate@workspace:extensions/dictate"],
|
||||
|
||||
"@jbcode/pi-effort-controls": ["@jbcode/pi-effort-controls@workspace:extensions/effort-controls"],
|
||||
|
||||
"@jbcode/pi-fast-browser": ["@jbcode/pi-fast-browser@workspace:extensions/fast-browser"],
|
||||
|
||||
"@jbcode/pi-file-search": ["@jbcode/pi-file-search@workspace:extensions/file-search"],
|
||||
|
||||
"@jbcode/pi-interactive-subagents": ["@jbcode/pi-interactive-subagents@workspace:extensions/interactive-subagents"],
|
||||
@@ -378,8 +392,6 @@
|
||||
|
||||
"@jbcode/pi-update": ["@jbcode/pi-update@workspace:extensions/update"],
|
||||
|
||||
"@jbcode/pi-workflow": ["@jbcode/pi-workflow@workspace:extensions/workflow"],
|
||||
|
||||
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="],
|
||||
|
||||
"@mariozechner/clipboard": ["@mariozechner/clipboard@0.3.9", "", { "optionalDependencies": { "@mariozechner/clipboard-darwin-arm64": "0.3.9", "@mariozechner/clipboard-darwin-universal": "0.3.9", "@mariozechner/clipboard-darwin-x64": "0.3.9", "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-musl": "0.3.9", "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" } }, "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA=="],
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
---
|
||||
status: accepted
|
||||
status: superseded
|
||||
superseded-by: 0006-skill-led-development-workflow-04SEP2026.md
|
||||
---
|
||||
|
||||
# Build workflow around persisted policy gates
|
||||
|
||||
> Superseded by [ADR 0006](./0006-skill-led-development-workflow-04SEP2026.md). Retained as the legacy custom-engine decision.
|
||||
|
||||
`workflow` will be a new extension rather than an activation or rename of `_workflows`. It will expose one command-style tool backed by atomic revisioned snapshots, Markdown Agent Profiles, deterministic eligibility followed by planner selection, a mandatory feature pipeline, and restricted feature or read-only analysis scripts. Every feature run receives a distinct exclusive worktree. Concurrency defaults to 8 under an immutable process-wide maximum of 32. Non-trivial UI work requires at least three structurally distinct prototypes and explicit user acceptance. Before implementation, the user chooses a fresh plan-seeded session, compaction of the current session that prefers `smart_compact` and falls back to normal Pi compaction when unavailable, or unchanged current context. Review uses a fresh session and excludes every Agent Profile that mutated the reviewed diff.
|
||||
|
||||
This keeps the caller interface small while making approval, prototype, recovery, and reviewer-independence rules enforceable in one workflow engine. We rejected unrestricted custom graphs because they could bypass policy, a general registry architecture because it adds speculative extension points, and event-sourced persistence because revisioned snapshots already provide the required recovery and audit data with less machinery.
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
---
|
||||
status: accepted
|
||||
status: superseded
|
||||
supersedes: 0001-workflow-architecture-24AUG2026.md
|
||||
superseded-by: 0006-skill-led-development-workflow-04SEP2026.md
|
||||
---
|
||||
|
||||
# Use drill-down overlays and bounded recovery
|
||||
|
||||
> Superseded by [ADR 0006](./0006-skill-led-development-workflow-04SEP2026.md). Retained as legacy custom-engine UI and recovery evidence.
|
||||
|
||||
This decision supersedes the dashboard, dirty-source, execution-timeout, and startup-ownership assumptions in [ADR 0001](./0001-workflow-architecture-24AUG2026.md). The persisted gated pipeline, worktree isolation, and independent-review decisions from ADR 0001 remain accepted.
|
||||
|
||||
## Context
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# Reuse standard agents through workflow role overlays
|
||||
|
||||
> Superseded by [ADR 0006](./0006-skill-led-development-workflow-04SEP2026.md). Retained as the legacy custom-engine agent-selection decision.
|
||||
|
||||
`workflow` will reuse standard `pi-interactive-subagents` definitions instead of duplicating their prompts and model settings in workflow-only agents. Explicit configuration maps standard agent names to workflow roles, while `workflow` retains ownership, tool-intersection, Gate, verification, and reviewer-independence policy. The default maps `scout` to Discovery and `worker` to Implementation; bundled profiles remain fallbacks for roles without a standard mapping. This keeps one source of truth for agent identity without allowing arbitrary prompt prose or agent names to imply mutability or workflow authority.
|
||||
|
||||
@@ -10,6 +10,8 @@ Publish packages to the `jb` owner endpoint of the Gitea npm registry. Repositor
|
||||
|
||||
`extensions/_workflows` is reference code, not a package or publishable extension. `@jbcode/pi-workflow` remains a private workspace until the required reference runtime is merged under `extensions/workflow`; this prevents publishing two workflow packages or publishing an incomplete package.
|
||||
|
||||
> **Later decision:** [ADR 0006](./0006-skill-led-development-workflow-04SEP2026.md) supersedes the workflow-specific merge and release direction above. Keep the legacy workspace private, unpublished, and unexposed during the skill pilot.
|
||||
|
||||
The existing `config/pi-links.json` policy remains the authority for live-development exposure. Package boundaries and publication do not automatically enable extensions globally.
|
||||
|
||||
## Consequences
|
||||
|
||||
@@ -23,3 +23,5 @@ Rename the private `pi-workflow` extension identity to `workflow`. Its workspace
|
||||
- Upstream movement is detected but never merged automatically; imported changes require reconciliation with monorepo-specific manifests and APIs.
|
||||
- Only `main` is tracked. Source tags, branches, and historical commits are not copied.
|
||||
- The renamed `workflow` extension remains private until its `_workflows` dependencies are merged and isolated packaging succeeds.
|
||||
|
||||
> **Later decision:** [ADR 0006](./0006-skill-led-development-workflow-04SEP2026.md) supersedes this workflow-specific merge and release direction with the skill pilot and a later retirement decision.
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
status: accepted
|
||||
supersedes:
|
||||
- 0001-workflow-architecture-24AUG2026.md
|
||||
- 0002-workflow-interaction-and-recovery-25AUG2026.md
|
||||
- 0002-workflow-standard-agent-overlays-27AUG2026.md
|
||||
---
|
||||
|
||||
# Replace the custom workflow engine with a skill-led workflow
|
||||
|
||||
> The legacy-runtime retention and pilot-before-retirement prerequisite below are superseded by [ADR 0007](0007-archive-legacy-workflows-05SEP2026.md). The approval, disposable-data, review, verification, and record-commit policies are superseded by [ADR 0008](0008-risk-based-development-workflow-07OCT2026.md). The skill-led architecture and outstanding validation obligations remain in force; interpret scenario expectations under the current policy.
|
||||
|
||||
## Context
|
||||
|
||||
The custom `workflow` extension encoded useful approval, isolation, verification, and reviewer-independence policy, but coupled that policy to a large state machine, child runtime, scheduler, persistence layer, and TUI. Recorded runs failed or stalled before implementation when exact planner output, child delivery, recovery state, or liveness mechanics diverged from the engine contract.
|
||||
|
||||
Pi already provides the useful orchestration primitives: `ask_user`, task tools, direct subagents, artifact presentation, prototypes, browser/application validation, project commands, and Git worktrees. Repository Markdown and commits provide a simpler recovery boundary than a second workflow runtime.
|
||||
|
||||
## Decision
|
||||
|
||||
Use the model-invoked [`develop-workflow` skill](../../skills/develop/develop-workflow/SKILL.md) for feature work and other multi-step changes. Keep questions and obvious low-risk edits on lightweight paths. Add no replacement extension.
|
||||
|
||||
The parent agent owns classification, requirements, the retained workflow record, task state, worktree, assignments, verification, commits, review, and local integration. Preserve categorical human gates for the exact implementation plan, non-trivial UI direction, and just-in-time risk. Require observable evidence rather than exact model-authored transition schemas.
|
||||
|
||||
Keep one committed Markdown workflow record per active change. Update it atomically at meaningful checkpoints with optimistic SHA-256 concurrency and retain it after completion. Treat Git and the last committed valid record as authoritative recovery evidence.
|
||||
|
||||
Use bounded subagent assignments with exclusive file ownership and before/after Git evidence. Exclude every mutator identity from independent standards and specification review. Require Git-capable read-only reviewer profiles to regenerate and confirm the pinned binary diff fingerprint. Reviewers report findings but never remediate them.
|
||||
|
||||
Automate verified commits, rebase, fresh review, clean local fast-forward integration, metadata-only closure, and safe cleanup. Keep destructive actions, force operations, conflict-driven behavior changes, pushes, publishing, deployment, and production access behind explicit approval.
|
||||
|
||||
Retain `extensions/workflow` and `extensions/_workflows` as private historical reference during the pilot. Keep them unexposed. Archive or remove them only in a separate change after three successful real-work pilots and explicit retirement approval.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Workflow gates become procedural and testable, not technically enforced by a custom state machine.
|
||||
- Recovery depends on repository evidence and proven child liveness rather than leases or automatic replay.
|
||||
- The skill composes existing Pi capabilities and adds no runtime dependency.
|
||||
- A Git-capable reviewer profile declares no edit/write tools, while parent before/after Git checks remain necessary because shell access can mutate indirectly.
|
||||
- Existing package and imported-snapshot ADRs remain valid except for workflow-extension-specific prerequisites, which this decision supersedes.
|
||||
- The twelve scenario evaluations and three real-change pilot remain required before retiring the legacy implementation or closing the replacement TODO.
|
||||
@@ -0,0 +1,5 @@
|
||||
# Archive the legacy workflow implementations
|
||||
|
||||
On 05SEP2026, JB explicitly approved archiving both legacy workflow implementations now rather than waiting for the three successful pilots required by [ADR 0006](0006-skill-led-development-workflow-04SEP2026.md). Move their source and tests to `__archived/extensions/workflow` and `__archived/extensions/_workflows`, outside active workspaces and test commands, while retaining historical documentation and the separate skill-validation TODO.
|
||||
|
||||
This supersedes only the retention and pilot-before-retirement prerequisite. It does not claim that scenario evaluation or pilots passed, change the `develop-workflow` skill, or authorize removal of historical run evidence. The archive preserves reference source rather than maintaining runnable legacy packages.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Use risk-based development workflow gates
|
||||
|
||||
On 07OCT2026, JB approved replacing routine plan-approval pauses and repeated full verification with risk-based automation. Clear work proceeds from a short recorded plan; unresolved product/design choices and meaningful risk retain human gates. This supersedes the categorical plan approval, migration approval, commit-per-gate, two-reviewer, and repeat-full-suite policies in [ADR 0006](0006-skill-led-development-workflow-04SEP2026.md) and its [implementation plan](../workflow/skill-led-development-workflow-plan.md).
|
||||
|
||||
A standing repository policy may authorize schema changes and reset/reseed for named, verified disposable environments. Pre-production alone grants no authority; valuable/shared data, production access, external compatibility, and uncertain targets remain protected. Prefer direct changes over speculative compatibility or rollback machinery when the actual preservation contract permits rebuilding.
|
||||
|
||||
Use focused checks while developing, one independent read-only reviewer covering standards and specification, targeted follow-up on bounded fixes, and one full required suite after remediation. Reuse evidence only while its inputs remain applicable; failures and broader changes still require appropriate reruns. Keep tests complete for meaningful behavior and credible failures rather than exhaustive permutations at every layer. Stricter repository/CI/hook requirements remain binding until explicitly revised.
|
||||
|
||||
Preserve worktrees, scoped authorization, non-trivial UI direction approval, real interface validation, mutator exclusion, pinned review evidence, recoverable records, and safe local integration. Batch record updates into meaningful commits rather than requiring initial, per-gate, and closure commits. Keep the existing checkpoint heading contract and archived engines; add no replacement extension or hook runtime changes.
|
||||
|
||||
The prior twelve-scenario and three-pilot validation obligation remains open in [todos](../todos.md#replace-workflow-with-a-skill-based-orchestrator), with expected decisions interpreted under this superseding policy. Focused policy walkthroughs for this revision are not claims that those execution scenarios or pilots passed.
|
||||
@@ -0,0 +1,206 @@
|
||||
# Local browser automation with Pi and Typesafe JEV
|
||||
|
||||
Status: feasibility review; proposed architecture, not an implementation decision.
|
||||
Reviewed: 2026-09-18. No browser tasks or inference benchmarks were run. No API keys were used or changed.
|
||||
|
||||
## Verdict
|
||||
|
||||
**Worth a bounded prototype for DOM-accessible workflows; not a demonstrated replacement for a general browser agent.** Browser-harness already supports local Chrome control without Browser Use Cloud. JEV can plausibly reduce the cost and latency of semantic choices between known actions. Its usefulness depends on how reliably code can expose the correct action and verify its outcome.
|
||||
|
||||
The most important correction: **JEV is text-only, not a screenshot-to-action model.** Use accessibility/DOM observations, not screenshots, on the JEV fast path. A vision-capable Pi model would be a separate fallback for visually grounded tasks.
|
||||
|
||||
A skill alone can teach Pi how to use the browser but does not remove main-model turns. The candidate design is a **Pi extension containing a bounded decision loop, with browser-harness as its local execution backend and a companion skill for workflow guidance**. A Chrome extension is not necessary for the initial experiment.
|
||||
|
||||
## What the sources establish
|
||||
|
||||
### Browser-harness
|
||||
|
||||
Inspected upstream commit `afbcc381b963040c19627d788e40c7e7663171ee`, package version `0.1.13`, MIT license.
|
||||
|
||||
- Local Chrome/Chromium is supported; cloud browsers are optional. The package requires Python >=3.11; its setup prompt recommends Python 3.12.
|
||||
- A persistent local daemon owns the CDP connection. Python helpers provide browser operations; the CLI executes Python, and an optional stdio MCP server exposes helpers as tools.
|
||||
- Upstream recommends accessibility-tree grounding using `Accessibility.getFullAXTree`, `backendDOMNodeId`, box-model coordinates, and outcome verification. Screenshots are a fallback when layout or imagery matters.
|
||||
- The default local daemon has one mutable current tab. Concurrent tab switching and actions can race. A Pi adapter needs serialized browser operations, not just per-request locking.
|
||||
- `fill_input()` emits key events per character; `type_text()` uses `Input.insertText`. Long-text entry can dominate latency even with a fast text model. Framework-managed inputs require event handling and exact-value verification; bulk insertion is not a universal replacement.
|
||||
- Arbitrary Python/JS and self-written helpers are powerful but inappropriate as the automatic JEV action interface. Expose vetted typed operations instead; keep any helper authoring in the supervised slow path.
|
||||
|
||||
Local-only configuration needs care:
|
||||
|
||||
- Set `BH_TELEMETRY=0`: telemetry is otherwise enabled by default.
|
||||
- Set `BH_UPDATE_CHECK=0`: the CLI otherwise checks PyPI for updates.
|
||||
- **Unset `BU_AUTOSPAWN`**, rather than setting it to `0`: source checks whether the environment value is nonempty.
|
||||
- Explicitly bind to the approved local browser. Do not expose remote-browser start, profile sync, cloud authentication, or proxy-fetch helpers in the fast adapter.
|
||||
- Do not supply a Browser Use key to this worker. `http_get()` selects a hosted fetch proxy when `BROWSER_USE_API_KEY` is present. Treat environment and workspace `.env` inheritance as part of the audit.
|
||||
- Keep recordings off unless requested (`BH_RECORD=0` is an explicit override). Treat saved DOM, screenshots, and logs as sensitive.
|
||||
|
||||
This establishes that hosted browser infrastructure is unnecessary, not that all dependencies have been exhaustively audited for outbound traffic. An implementation should verify network egress explicitly.
|
||||
|
||||
### Typesafe JEV
|
||||
|
||||
The documented API is `POST https://api.typesafe.ai/v1/systemone`, authenticated with a bearer key. Requests contain `state`, `model`, and named `questions`.
|
||||
|
||||
- Current documented version: `jev-1.13.0`. Pin a version when calibrating thresholds; `jev-latest` can move.
|
||||
- Inputs: text or structured JSON; no images, audio, or video.
|
||||
- Outputs: `Choice` among supplied options, `Noul` yes-probability, or `Score` against supplied criteria. This is not arbitrary prose, code, or a general agent action stream.
|
||||
- Multiple questions about the same state are evaluated independently in parallel. A question cannot rely on another question's answer in the same request.
|
||||
- Published limits: 64k tokens for the request and 32k for state plus the longest question. These are ceilings, not recommended browser-state sizes.
|
||||
- Published price: $0.042 per million input tokens, with output tokens free. At 2,000 input tokens per request, 1,000 decisions would cost approximately $0.084 for JEV alone, assuming that token count matches billing. Planner/text calls and retries are additional.
|
||||
- Typesafe says most queries complete in about 100 ms. **This is a provider claim, not measured local end-to-end latency or a browser-task guarantee.**
|
||||
- The docs say customer requests/responses are not used for training. They mention enterprise ZDR separately; do not assume a normal API account has zero retention.
|
||||
- The JavaScript SDK is `@typesafe-ai/sdk` (Node.js >=20), using `TypeSafeClient.systemOne()` and `TYPESAFE_API_KEY`. Keep this as an independent API adapter, not a fake chat model.
|
||||
- Typesafe explicitly documents weakness with numeric precision, counting, date comparison, multi-hop indirection, irrelevant context, and adversarial input. Parse/count/compare in code; give JEV semantic labels and short, direct questions.
|
||||
- The documented adversarial-content limitation is especially relevant to websites: page content can steer classifications. Instructions and confidence thresholds alone are not adequate protection.
|
||||
- Choice confidence and Noul probabilities are not interchangeable. Calibrate each gate separately; a relative Choice can pick a winner even when every option is unsuitable.
|
||||
|
||||
Local browser execution still sends selected page-derived text to Typesafe and generated-text context to the configured Pi model provider. Redact passwords, tokens, cookies, and unrelated page content before those calls.
|
||||
|
||||
## Proposed execution split
|
||||
|
||||
```text
|
||||
User goal -> Pi main model: bounded subgoal and success criteria
|
||||
|
|
||||
Pi extension
|
||||
|
|
||||
observe AX/DOM, filter candidates
|
||||
|
|
||||
exact deterministic match available?
|
||||
yes: use code no: JEV Choice
|
||||
|
|
||||
policy + freshness + confidence checks
|
||||
|
|
||||
typed local browser-harness bridge
|
||||
|
|
||||
execute -> verify -> repeat
|
||||
|
|
||||
finished / uncertain / budget exhausted
|
||||
|
|
||||
return to Pi
|
||||
|
||||
Text needed: supplied literal -> insert directly
|
||||
new prose -> configured small Pi model -> validate -> insert
|
||||
Visual or novel reasoning needed -> return to Pi, optionally with screenshot
|
||||
```
|
||||
|
||||
### Responsibilities
|
||||
|
||||
| Component | Responsibility |
|
||||
| --- | --- |
|
||||
| Pi main model | Interpret novel goals, supply bounded subgoals, handle recovery requiring reasoning |
|
||||
| Extension controller | Budgets, state machine, permissions, cancellation, model routing, traces |
|
||||
| DOM/AX adapter | Extract relevant controls and labels; generate candidate IDs tied to tab/frame/document |
|
||||
| JEV | Rank/select candidates, classify known states, flag uncertainty, assess narrow semantic predicates |
|
||||
| Small Pi model | Generate genuinely new text within supplied facts and constraints |
|
||||
| Browser bridge | Execute allowlisted operations; verify target identity and resulting browser state |
|
||||
| Companion skill | Explain usage, escalation rules, and reusable site-specific workflows |
|
||||
|
||||
JEV should **replace a main-model microdecision**, not be added before an unchanged main-model turn. The deterministic path should bypass JEV entirely when no semantic judgment is needed.
|
||||
|
||||
Candidate sets must include `none_of_the_above`/`needs_more_context`. Selecting the best of an incomplete set can still be wrong. Keep candidate descriptions grounded in visible labels, roles, nearby headings and form structure; filter hidden/disabled elements and preserve frame identity. Treat the correct-target inclusion rate as a first-class metric.
|
||||
|
||||
Run independent page-state questions together, but do not confuse speculative decisions with permission to execute alternative actions in parallel. Reobserve after navigation or meaningful DOM changes. Invalidate stale candidates before any click; do not cache coordinates across page transitions.
|
||||
|
||||
For literals such as a supplied email address, search phrase, or exact message, **no text-model call is necessary**. A generation model is only justified for composition or rewriting. Validate length, exact required facts, and field-specific constraints. Do not regenerate text when retrying insertion; verify the actual value and framework state.
|
||||
|
||||
### Scope and limitations
|
||||
|
||||
Good first targets: locating a labeled control, navigating a known menu, choosing a relevant result from a shortlist, filling conventional forms, recognizing a success/error message, and repeating established workflows.
|
||||
|
||||
Weak targets: canvas-only applications, image-only controls, novel workflows that require broad planning, CAPTCHAs, ambiguous authorization, and long tasks where the correct action is not represented in the candidate set. Shadow DOM, cross-origin frames, rich-text editors, and changing layouts need explicit coverage rather than assumptions.
|
||||
|
||||
## Pi integration findings
|
||||
|
||||
The installed Pi API supports this split without changing the main conversation model:
|
||||
|
||||
- Register typed tools with `pi.registerTool`; tool execution receives an abort signal and extension context.
|
||||
- Resolve the auxiliary text model with `ctx.modelRegistry.find(provider, id)` and check `getAvailable()` / `hasConfiguredAuth(model)` at runtime.
|
||||
- Call `ctx.modelRegistry.complete(model, context, { signal })` for independent text generation. This uses Pi's model registry rather than duplicating provider credential handling.
|
||||
- Keep Typesafe behind its own SDK/HTTP adapter; its constrained decision API does not need a Pi chat-provider registration.
|
||||
- Cancellation must propagate to model calls and the local worker. Stopping a tool does not undo an already executed browser action.
|
||||
|
||||
Sanitized cached model metadata includes these **candidates, not verified live availability or measured speed rankings**:
|
||||
|
||||
| Candidate | Potential role |
|
||||
| --- | --- |
|
||||
| `anthropic/claude-haiku-4-5` | Small text-generation baseline; metadata also advertises image input |
|
||||
| `anthropic/claude-haiku-4-5-20251001` | Pinned Haiku alternative for reproducibility |
|
||||
| `openai-codex/gpt-5.3-codex-spark` | Text-only alternative to benchmark if runtime authorization permits |
|
||||
|
||||
Start by comparing Haiku and Spark on short, constrained field-generation tasks; do not select a winner from model names alone. A local LM Studio provider is another supported integration route, but no particular loaded local model was verified.
|
||||
|
||||
Relevant precedents:
|
||||
|
||||
- Installed Pi `examples/extensions/qna.ts` and `summarize.ts`: independent registry completions and cancellation.
|
||||
- `extensions/lm-studio/index.ts`: dynamic local OpenAI-compatible model discovery.
|
||||
- `extensions/subagents/src/backends/pi.ts`: registry reuse and abort lifecycle handling.
|
||||
- Adjacent project `/srv/work/personal/code/eden/src/lib/server/assistant/intent.ts` and `tests/intent.test.ts`: JEV Choice routing with pinned version, response validation, confidence gating and failure-closed tests. This is a reference, not a proposed runtime dependency.
|
||||
|
||||
Browser-harness exposes Python/CLI/MCP interfaces; no existing TypeScript-native bridge was identified in this investigation. A restricted local Python bridge is the proposed adapter, with the existing MCP server as an alternative transport to evaluate. Keep the controller independent of transport so this choice can change without rewriting decision logic. Measure worker startup/IPC before committing to an additional always-running service.
|
||||
|
||||
## Safety and reliability requirements
|
||||
|
||||
- Code, not model confidence, enforces allowed domains and actions. Treat all page text as untrusted data, never instructions for the controller.
|
||||
- Stop for approval before purchases, sending messages, publishing, destructive changes, granting access, or other consequential submissions unless the exact action is already authorized.
|
||||
- Keep credentials out of model inputs. Stop for passwords, MFA, and ambiguous account selection; prefer an explicitly approved logged-in test profile.
|
||||
- Bind candidates and approval to the precise browser/tab/frame, target identity, intended action and payload. Revalidate immediately before execution.
|
||||
- Do not blindly retry non-idempotent actions after timeouts. Inspect the resulting state first: a cancelled request may already have clicked or submitted.
|
||||
- Bound steps, elapsed time, model requests, and recovery attempts. Detect repeated states/no progress, release the browser lane on stop, and surface a useful partial result.
|
||||
- Fail closed on malformed responses, missing targets, expired observations, and permission uncertainty. Semantic confidence is not a security boundary.
|
||||
- Keep the browser debug endpoint local/private. Starting a browser on a remote Pi host does not automatically attach to a desktop browser elsewhere; deployment location needs confirmation before a prototype.
|
||||
|
||||
## Benchmark before committing to the architecture
|
||||
|
||||
Use a shared local browser backend and comparable observations, test fixtures, warm/cold conditions, and safety policy. Pin model versions and prompts. No live purchases, messages, or destructive tasks in the initial suite.
|
||||
|
||||
Compare four paths:
|
||||
|
||||
1. Main Pi model chooses each browser action: conventional baseline.
|
||||
2. Small Pi model chooses from the same bounded candidate list.
|
||||
3. JEV chooses from that list; small model only generates text when needed.
|
||||
4. Deterministic/site-specific workflow where applicable: establishes whether AI adds value at all.
|
||||
|
||||
Use 30–50 task instances across navigation, result selection, conventional forms, generated text, and recovery. Repeat each several times, randomize run order, reset initial state, and report uncertainty. This is a screening study, not proof of general-web reliability. Use held-out pages/tasks for confirmation.
|
||||
|
||||
Measure:
|
||||
|
||||
- Verified task success, critical misactions, and human interventions.
|
||||
- End-to-end p50/p95 latency and time-to-verified-success, including failed runs/timeouts rather than reporting only successful fast runs.
|
||||
- Observe/filter time, payload size, API RTT, action time, DOM settling, verification, text insertion, and escalation latency separately.
|
||||
- Candidate recall, selected-action accuracy, no-progress loops, stale-target failures, and fallback frequency.
|
||||
- Main-model calls eliminated, token usage and total cost per verified successful task.
|
||||
- Confidence versus empirical action accuracy, by task type. Tune thresholds on calibration data and evaluate on held-out data; do not assume a probability cutoff guarantees browser reliability.
|
||||
|
||||
Illustrative latency budget, **not a measurement**:
|
||||
|
||||
`T_task = planning + Σ(observe + decide + act + settle + verify) + text generation + recovery`
|
||||
|
||||
Replacing a 1,500 ms decision with 200 ms when all other step work takes 500 ms changes a step from 2,000 ms to 700 ms (~2.9× faster). If the site instead spends 3,000 ms loading, the same replacement gives only ~1.4×. Retry and escalation costs can erase either gain.
|
||||
|
||||
Suggested promotion gate for discussion: roughly 2× median improvement over the main-model baseline on the target workflows, no observed critical misactions, task success comparable to baseline, and a material advantage over the small-model-only alternative. Tail latency and fallback cost must remain acceptable. These are proposed targets, not established outcomes or statistical guarantees.
|
||||
|
||||
## Suggested next experiment
|
||||
|
||||
Build only after agreeing on the architecture and deployment target:
|
||||
|
||||
1. An offline AX/DOM fixture adapter and JEV-vs-small-model action-selection benchmark. First establish candidate recall and decision quality without any side effects.
|
||||
2. A bounded local execution prototype using the same candidate schema, with literal form filling and deterministic postconditions.
|
||||
3. Add generated-text calls and supervised escalation only if the first two stages show a useful advantage.
|
||||
|
||||
Do not begin with a general autonomous browser agent, self-modifying helpers, Chrome extension, hosted browser, or custom JEV chat-provider emulation.
|
||||
|
||||
## Sources
|
||||
|
||||
- [Browser-harness README](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/README.md)
|
||||
- [Browser workflow and local concurrency](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/src/browser_harness/SKILL.md)
|
||||
- [CDP and input helpers](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/src/browser_harness/helpers.py)
|
||||
- [Telemetry controls](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/src/browser_harness/telemetry.py), [cloud autospawn](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/src/browser_harness/run.py), [update checks](https://github.com/browser-use/browser-harness/blob/afbcc381b963040c19627d788e40c7e7663171ee/src/browser_harness/admin.py)
|
||||
- [Typesafe workflow architecture](https://docs.typesafe.ai/concepts/how-to-build-with-system-one)
|
||||
- [JEV models, pricing, limits and data handling](https://docs.typesafe.ai/models)
|
||||
- [Text-only state contract](https://docs.typesafe.ai/concepts/state)
|
||||
- [Typesafe HTTP API](https://docs.typesafe.ai/api)
|
||||
- [JEV 1.13 documented limitations](https://docs.typesafe.ai/model-jaggedness/jev-1.13)
|
||||
- [Typesafe JavaScript SDK](https://docs.typesafe.ai/sdk/javascript)
|
||||
- Installed Pi API reference: `/home/jb/.bun/install/global/node_modules/@earendil-works/pi-coding-agent/dist/core/model-registry.d.ts`
|
||||
- Installed Pi extension documentation: `/home/jb/.bun/install/global/node_modules/@earendil-works/pi-coding-agent/docs/extensions.md`
|
||||
- Installed Pi auxiliary-model examples: `/home/jb/.bun/install/global/node_modules/@earendil-works/pi-coding-agent/examples/extensions/{qna,summarize}.ts`
|
||||
|
||||
Research inputs were cached under `/tmp/browser-efficacy/.firecrawl/`; the inspected upstream checkout is `/tmp/browser-efficacy/browser-harness/`. These temporary caches are not deliverables or durable repository dependencies.
|
||||
@@ -0,0 +1,415 @@
|
||||
# Configurable Dictate ASR backends
|
||||
|
||||
Status: implemented and validated, 03SEP2026
|
||||
|
||||
## Decisions
|
||||
|
||||
- Keep microphone capture and every ASR implementation in the existing host-side `pi-dictate-service` boundary.
|
||||
- Keep provider settings and API keys in host-owned, mode-`0600` configuration outside Pi's nono sandbox. The Pi extension must never read or transmit cloud credentials.
|
||||
- Select one backend in host configuration and apply the change by restarting the socket-activated service. Do not add per-session or per-generation provider selection to the Pi protocol.
|
||||
- Support three backend profiles:
|
||||
- local NeMo/Parakeet;
|
||||
- Meta Model API Muse Voice Transcribe;
|
||||
- OpenAI Realtime transcription.
|
||||
- Stream microphone audio to both cloud providers while recording. Preserve the current push-to-start, push-to-stop interaction and paste only one finalized transcript after stop.
|
||||
- Keep local as the migration default. Cloud audio and cost require an explicit backend selection and credential setup.
|
||||
- Default new OpenAI configuration to `gpt-live-transcribe`. Permit a configurable OpenAI transcription model, and document `gpt-transcribe` as the commit-oriented alternative. Do not make `whisper-1` the default: current OpenAI guidance recommends the GPT transcription models, and `whisper-1` is not the selected realtime path.
|
||||
|
||||
## Goals
|
||||
|
||||
1. A user can configure one local, Meta, or OpenAI ASR backend without changing extension source.
|
||||
2. Local mode retains its offline runtime, pinned-model verification, coding dictionary, and network-denied systemd sandbox.
|
||||
3. Meta and OpenAI modes connect before recording, stream bounded PCM without writing audio files, finalize on stop, and return one bounded transcript.
|
||||
4. The extension continues to see only bounded state, level, result, and error events over `/run/user/1000/pi-dictate.sock`.
|
||||
5. API keys, provider wire events, partial hypotheses, and raw audio never cross into Pi.
|
||||
6. Switching backend is fail-closed: configuration, installed dependency profile, and systemd network hardening must agree before the service starts.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Automatic failover between providers.
|
||||
- Sending one recording to multiple providers.
|
||||
- A runtime backend picker in Pi.
|
||||
- Partial text in Pi's editor.
|
||||
- Voice-activity-controlled submission; `Alt+M` remains the turn boundary.
|
||||
- Diarized editor output, timestamps, confidence scores, or translation.
|
||||
- Arbitrary local inference runtimes in the first slice. The local adapter remains NeMo-compatible; its model archive, checksum, terms file, and boost can become configuration.
|
||||
- Reusing credentials from Pi's provider auth store. Dictate gets dedicated host credentials so the service does not need access to all of `~/.pi/agent/auth.json`.
|
||||
|
||||
## Current architecture and constraints
|
||||
|
||||
- `extensions/dictate/index.ts` is a strict protocol-v2 Unix-socket client. It owns no audio, model, or network capability and pastes only a current finalized result.
|
||||
- `services/dictate/src/pi_dictate_service/server.py` serializes one generation globally and currently assumes a lazy `TranscriptionModel` with `load()` followed by `transcribe(samples)` after recording.
|
||||
- `services/dictate/src/pi_dictate_service/recorder.py` launches `pw-record` at 16 kHz mono PCM16, drains stdout/stderr in threads, retains at most five minutes of PCM, and derives the local level meter.
|
||||
- `services/dictate/src/pi_dictate_service/model.py` supports one explicit `.nemo` archive, verifies its SHA-256, applies the coding dictionary, and runs offline on CUDA.
|
||||
- `systemd/user/pi-dictate.service` denies all IP networking and limits address families to `AF_UNIX`. That must remain true in local mode but cannot be shared unchanged by cloud mode.
|
||||
- `services/dictate/install.sh` always installs the full NeMo/CUDA environment and pinned model. Cloud-only installation should not require either.
|
||||
- `profiles/nono/pi-local.json` already grants the exact Dictate Unix socket and explicitly denies the installed local runtime. It is policy-controlled and must remain unchanged for this feature; no Pi-side network, credential, PipeWire, model, or device grant is required.
|
||||
|
||||
## External API contract baseline
|
||||
|
||||
The implementation must turn the current provider documentation into checked fixtures before production code is merged, because Muse Voice Transcribe was released on the date of this plan and its contract may still move.
|
||||
|
||||
### Meta Muse Voice Transcribe
|
||||
|
||||
Primary references:
|
||||
|
||||
- [Meta announcement](https://research.meta.ai/blog/introducing-muse-voice-transcribe)
|
||||
- [Meta speech-to-text guide](https://ai.developer.meta.com/docs/speech-to-text)
|
||||
- [Meta realtime voice API reference](https://ai.developer.meta.com/docs/api-reference/voice/realtime)
|
||||
- [Meta voice event schemas](https://ai.developer.meta.com/docs/api-reference/voice/schemas)
|
||||
- [Meta Model API authentication](https://ai.developer.meta.com/docs/getting-started/authentication)
|
||||
- [Meta voice cookbook](https://github.com/meta-models/meta-model-cookbook/blob/main/06_muse_voice/01_voice_api_fundamentals/README.md)
|
||||
|
||||
Current documented shape:
|
||||
|
||||
- WebSocket endpoint: `wss://api.meta.ai/v1/asr/realtime`, with an optional `sessionId` query parameter.
|
||||
- Model: `muse-voice-transcribe-1.0`; no omitted-model default is documented.
|
||||
- The first text frame is a handshake requiring `authorization.accessToken`, `audioEncoding`, and `model`. The key value is `Bearer <MODEL_API_KEY>` inside that object; an HTTP authorization header or API-key query parameter is ignored.
|
||||
- The public API accepts signed little-endian mono PCM16 as binary frames. Use native `PCM_24KHZ` (48,000 bytes/second); `PCM_16KHZ` is also accepted and server-resampled.
|
||||
- `PUSH_TO_TALK` is the mode matching Dictate's one explicit turn. Defaults also include `partialMode: "CUMULATIVE"` and `emitAudioProgress: true`.
|
||||
- Wait for the server's `{ "sessionId": "..." }` acknowledgement before sending audio. Frames carry no headers, timestamps, sequence numbers, or semantic boundaries; approximately 80 ms pacing is guidance rather than a required frame size.
|
||||
- End input with the text frame `{ "type": "endStream" }` and require a `transcript` event with `final: true`. Treat that correlated final as completion and close the connection rather than waiting for Meta's idle close, while preserving any provider failure that races with finalization.
|
||||
- Cumulative partials replace the prior hypothesis. Dictate ignores partials, turn boundaries, endpointing, and speaker events and retains only the explicit bounded final transcript.
|
||||
- `keywords` and `languageBias` are handshake-only hints. The existing coding dictionary can feed `keywords`; no free-form prompt field is documented.
|
||||
- Meta documents a 60-minute session, at most five seconds of audio backlog with approximately realtime pacing, eight concurrent streams, and 1,000 streams/hour. Dictate's own five-minute cap remains stricter.
|
||||
- Treat close codes `1008` (invalid request/pacing), `1011` (backend or duration), and `1013` (rate limit), plus bounded `error` events, as provider failures.
|
||||
|
||||
The contract spike must freeze these fields as sanitized fixtures and confirm them against a live non-sensitive sample, especially because Muse Voice Transcribe was released on the date of this plan. Do not copy the undocumented `shortwave.facebook.com` Muse Code endpoint used by third-party extensions.
|
||||
|
||||
### OpenAI Realtime transcription
|
||||
|
||||
Primary references:
|
||||
|
||||
- [Realtime transcription guide](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||
- [Realtime WebSocket guide](https://developers.openai.com/api/docs/guides/realtime-websocket)
|
||||
- [Speech-to-text guide](https://developers.openai.com/api/docs/guides/speech-to-text)
|
||||
- [`gpt-live-transcribe` model](https://developers.openai.com/api/docs/models/gpt-live-transcribe)
|
||||
|
||||
Current documented shape:
|
||||
|
||||
- Use the GA server-side WebSocket connection `wss://api.openai.com/v1/realtime?intent=transcription` with `Authorization: Bearer <key>`. Do not mix this with the deprecated `/v1/realtime/transcription_sessions` schema or put the transcription model in the connection query.
|
||||
- After `session.created`, send the nested GA `session.update` shape with `type: "transcription"`, the configured transcription model, 24 kHz mono PCM16, and `turn_detection: null`; wait for `session.updated` before capture.
|
||||
- PCM is base64 encoded in bounded `input_audio_buffer.append` JSON events. The API's per-append limit is 15 MiB, but Dictate must use much smaller realtime chunks.
|
||||
- Stop sends `input_audio_buffer.commit`; it starts transcription without creating an assistant response.
|
||||
- Partial text arrives as `conversation.item.input_audio_transcription.delta`; final text arrives as `conversation.item.input_audio_transcription.completed` and is correlated by `item_id`. Transcription-specific failures use `conversation.item.input_audio_transcription.failed`; general protocol failures use `error`.
|
||||
- `gpt-live-transcribe` is the recommended low-latency model and supports prompt, keywords, languages, and bounded delay settings. A credentialed live spike on 03SEP2026 confirmed `delay: "low"`.
|
||||
- `gpt-transcribe` can use the same Realtime session but begins transcription after commit and can return detected languages.
|
||||
- `whisper-1` remains a file-transcription model for older integrations, translation, and timestamp use cases; its model page says realtime transcription is unsupported.
|
||||
- OpenAI documents a 60-minute Realtime session. Dictate's own five-minute cap remains stricter.
|
||||
- The current guide recommends `delay` for `gpt-live-transcribe`, while one API reference associates it with `gpt-realtime-whisper`. The live GA session accepted `low`; Dictate permits `minimal`, `low`, `medium`, `high`, and `xhigh`, and otherwise omits the field.
|
||||
|
||||
## Proposed host configuration
|
||||
|
||||
Add `services/dictate/src/pi_dictate_service/config.py` with strict, versioned parsing. The installed authority is:
|
||||
|
||||
```text
|
||||
~/.config/pi-dictate/
|
||||
config.json
|
||||
credentials/
|
||||
meta-api-key
|
||||
openai-api-key
|
||||
```
|
||||
|
||||
Illustrative schema (field names may be refined during implementation):
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"backend": "openai",
|
||||
"idleSeconds": 900,
|
||||
"local": {
|
||||
"adapter": "nemo-parakeet",
|
||||
"modelPath": "~/.local/share/pi-dictate/models/parakeet.nemo",
|
||||
"modelSha256": "...",
|
||||
"termsPath": "~/.local/share/pi-dictate/config/coding-terms.txt",
|
||||
"boostAlpha": 1.0
|
||||
},
|
||||
"meta": {
|
||||
"model": "muse-voice-transcribe-1.0",
|
||||
"apiKeyFile": "~/.config/pi-dictate/credentials/meta-api-key",
|
||||
"termsPath": "~/.local/share/pi-dictate/config/coding-terms.txt"
|
||||
},
|
||||
"openai": {
|
||||
"model": "gpt-live-transcribe",
|
||||
"apiKeyFile": "~/.config/pi-dictate/credentials/openai-api-key",
|
||||
"delay": "low",
|
||||
"prompt": "Coding dictation",
|
||||
"termsPath": "~/.local/share/pi-dictate/config/coding-terms.txt",
|
||||
"languages": ["en"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Configuration rules:
|
||||
|
||||
- Reject unknown root and selected-backend keys, malformed JSON, unsupported versions, invalid numeric bounds, and incompatible options.
|
||||
- Read only the selected backend's credential. Never require or expose credentials for inactive backends.
|
||||
- Credential files must be regular files owned by the service user, not symlinks, with no group/other permission bits and a small byte limit. Trim one trailing newline; reject empty or multi-line values.
|
||||
- Never accept literal keys on command lines, in systemd environment variables, in the Pi protocol, or in logs.
|
||||
- Keep provider endpoints fixed in production. Inject transports in tests rather than adding a credential-exfiltration-prone endpoint override. If a localhost override is needed for an opt-in integration harness, gate it behind an explicit test-only setting.
|
||||
- Keep the current pinned Parakeet revision as the generated local default. A custom local archive must supply an explicit checksum and use a supported NeMo adapter.
|
||||
- Validate coding terms against the existing byte/count limits before mapping them to provider keyword limits. Provider-specific filtering must reject unsupported characters instead of silently altering terms.
|
||||
- Configuration is read at process startup. Changes require the documented configure operation and service restart.
|
||||
|
||||
## Backend-neutral service design
|
||||
|
||||
Replace the model-only assumption with a backend that creates one prepared turn:
|
||||
|
||||
```text
|
||||
TranscriptionBackend
|
||||
descriptor # bounded backend/model diagnostics
|
||||
start_turn() -> Turn # load/reuse local model or connect/auth cloud session
|
||||
|
||||
TranscriptionTurn
|
||||
sample_rate_hz
|
||||
retain_pcm
|
||||
enqueue_pcm(chunk) # nonblocking, bounded
|
||||
finish(capture) -> text # infer locally or commit/await cloud final
|
||||
poll_error()
|
||||
cancel()
|
||||
close()
|
||||
```
|
||||
|
||||
`start_turn()` and `finish()` run outside the socket loop. A local backend persists its loaded model across turns; each cloud turn owns its own WebSocket and provider workers.
|
||||
|
||||
Exact Python protocols can differ, but the following invariants are mandatory:
|
||||
|
||||
1. `start_turn()` completes before `pw-record` starts. The UI must never say `listening` before provider auth/handshake succeeds.
|
||||
2. The backend declares the recorder rate: 16 kHz for the existing local adapter and 24 kHz for Meta/OpenAI.
|
||||
3. Local mode retains bounded PCM for post-stop inference. Cloud mode streams through a bounded queue and does not retain the whole recording.
|
||||
4. The recorder drain thread must never perform network I/O. It enqueues PCM quickly; a provider worker drains the queue.
|
||||
5. Queue overflow or a stalled provider is a terminal error. Stop capture and discard the generation; never silently drop audio and return a plausible but incomplete transcript.
|
||||
6. On stop, signal `pw-record`, drain stdout to EOF, wait for child close, then enqueue the provider end/commit marker after all PCM. This preserves the existing no-truncated-tail guarantee.
|
||||
7. Ignore partial hypotheses at the service boundary. Only an explicit provider final event may produce the protocol `result`.
|
||||
8. Every provider event, transcript, close reason, and error is byte-bounded before parsing or logging.
|
||||
9. Cloud transports use normal certificate and hostname verification, fixed production endpoints, no credential-bearing redirects, and no ambient proxy environment unless a separately reviewed proxy feature is added.
|
||||
10. Cancellation/disconnect tombstones delivery immediately, stops capture, closes the provider turn, joins worker threads with bounded timeouts, and suppresses every later final event.
|
||||
11. The service remains globally single-generation. Idle exit is suspended during local load/inference and cloud connect/stream/finalization.
|
||||
|
||||
### Lifecycle
|
||||
|
||||
The existing extension states remain valid:
|
||||
|
||||
```text
|
||||
idle -> connecting -> recording -> stopping -> transcribing -> idle
|
||||
```
|
||||
|
||||
Service interpretation becomes backend-neutral:
|
||||
|
||||
- `loading`: preparing the selected backend (local model load or cloud connect/auth/session setup);
|
||||
- `listening`: recorder and, for cloud mode, provider stream are both ready;
|
||||
- `stopping`: recorder shutdown and complete PCM drain;
|
||||
- `transcribing`: local inference or cloud commit/end-of-input plus final-result wait;
|
||||
- `idle`: generation resources are closed and no result remains deliverable.
|
||||
|
||||
Change extension text from `loading model...` to `preparing transcription...`.
|
||||
|
||||
## Provider adapters
|
||||
|
||||
### Local NeMo/Parakeet
|
||||
|
||||
Refactor `NemoParakeetModel` behind `LocalNemoBackend` without changing its observable transcription behavior:
|
||||
|
||||
- lazy checksum-verified load;
|
||||
- 16 kHz PCM16 capture and current five-minute/9.6 MB bound;
|
||||
- offline environment and CUDA behavior;
|
||||
- coding-term phrase boosting;
|
||||
- post-stop conversion and inference;
|
||||
- warm model reuse until idle service exit.
|
||||
|
||||
This slice must pass the existing service tests before either cloud adapter is introduced.
|
||||
|
||||
### Meta
|
||||
|
||||
Add `backends/meta.py` and a small bounded WebSocket transport:
|
||||
|
||||
- read and validate the dedicated Meta key;
|
||||
- connect only to the public `api.meta.ai` realtime endpoint;
|
||||
- send the documented first-frame bearer handshake for `muse-voice-transcribe-1.0`, `PCM_24KHZ`, `PUSH_TO_TALK`, and `CUMULATIVE` partials;
|
||||
- wait for the `sessionId` acknowledgement before starting capture;
|
||||
- send recorder bytes as ordered, approximately realtime-paced binary frames and enforce Meta's five-second backlog limit locally;
|
||||
- consume but do not expose cumulative partial hypotheses;
|
||||
- send `{ "type": "endStream" }` after recorder EOF;
|
||||
- require a `transcript` event with `final: true`, then consume the normal `1000` close;
|
||||
- map bounded provider error events and close codes `1008`, `1011`, and `1013`, plus auth, timeout, malformed-event, and premature-close failures, to stable `meta_*` service error codes without including the key or raw provider payload.
|
||||
|
||||
Diarization and endpointing events are parsed defensively and ignored for editor delivery in this feature.
|
||||
|
||||
### OpenAI
|
||||
|
||||
Add `backends/openai.py`:
|
||||
|
||||
- connect to the GA `intent=transcription` URL with the standard key in the `Authorization` header, never the URL;
|
||||
- wait for `session.created`, send the nested GA `session.update` configuration with 24 kHz PCM and `turn_detection: null`, then wait for `session.updated` before capture;
|
||||
- emit bounded `input_audio_buffer.append` events with base64 PCM, far below the provider's 15 MiB event limit;
|
||||
- commit only after recorder EOF;
|
||||
- correlate final completion to the committed item and ignore stale/unmatched events;
|
||||
- accept configurable model, prompt, keywords, languages, and the live-verified bounded `delay` values; reject unknown values rather than silently omitting them.
|
||||
- default to `gpt-live-transcribe`; document and test `gpt-transcribe`'s commit-oriented behavior. Omit `delay` when it is not configured so the provider default remains authoritative.
|
||||
- map `conversation.item.input_audio_transcription.failed`, general `error` events, auth/rate limits, timeouts, malformed events, and premature closes to stable `openai_*` errors without logging request bodies or keys.
|
||||
|
||||
Do not build around the OpenAI SDK unless it materially simplifies Realtime event handling without hiding cancellation, queue bounds, or frame limits. A direct WebSocket adapter is likely smaller and easier to fixture-test.
|
||||
|
||||
## Pi protocol and extension work
|
||||
|
||||
Preserve protocol version 2 unless the implementation discovers a required breaking frame change. The existing state/result contract already represents all three backends.
|
||||
|
||||
Additive changes are sufficient:
|
||||
|
||||
- Allow `pong` to report bounded `backend` and `model` strings for diagnostics. Never include paths, endpoints with query data, or credential state.
|
||||
- Define `ready` as whether the selected backend can begin immediately without preparation; the extension must not depend on this field for state transitions.
|
||||
- Add `/dictate-status` to establish a short hello/ping when idle and report selected backend/model. It is read-only and does not expose host config or credentials.
|
||||
- Keep `Alt+M`, `Alt+N`, `/dictate-cancel`, stale-generation rejection, transcript bounds, and `pasteToEditor()` behavior unchanged.
|
||||
- Do not add a Pi-side configuration writer. Backend changes are privileged host operations outside the agent sandbox.
|
||||
|
||||
If additive diagnostic fields are not worth the extra command/client complexity, omit them rather than bumping the protocol solely for display.
|
||||
|
||||
## Installer, packaging, and systemd
|
||||
|
||||
### Dependency profiles
|
||||
|
||||
Split `services/dictate/pyproject.toml` into a small core plus locked optional profiles:
|
||||
|
||||
- core: protocol, recorder, configuration, and service code;
|
||||
- local extra: NeMo, NumPy, PyTorch, and current CUDA stack;
|
||||
- cloud extra: the selected maintained WebSocket client and its locked dependencies.
|
||||
|
||||
Cloud-only installation must not download the Parakeet archive or install the multi-gigabyte local CUDA stack. Local installation remains reproducible and checksum pinned.
|
||||
|
||||
### Configure operation
|
||||
|
||||
Extend `services/dictate/install.sh` (or add an installer-owned companion command) with explicit operations such as:
|
||||
|
||||
```text
|
||||
install.sh install --backend local [--model-source PATH]
|
||||
install.sh install --backend meta --api-key-file PATH
|
||||
install.sh install --backend openai --api-key-file PATH
|
||||
install.sh configure --backend ...
|
||||
install.sh check
|
||||
```
|
||||
|
||||
- `--api-key-file` copies from a file into the fixed credential directory with mode `0600`; never accept a literal secret argument.
|
||||
- Write config and generated systemd hardening atomically, run validation, daemon-reload, and restart the service/socket as one operation.
|
||||
- Preserve existing config, keys, coding dictionary, and valid model archives on reinstall unless an explicit replacement option is supplied.
|
||||
- Make uninstall's treatment of `~/.config/pi-dictate` explicit; prefer preserving credentials by default with a separate `--purge-config` confirmation path.
|
||||
- Resolve the existing installer model-reuse todo while this flow is being changed: checksum and reuse a valid installed archive before attempting a network download.
|
||||
|
||||
### Backend-specific network hardening
|
||||
|
||||
Do not weaken local mode to make cloud mode work.
|
||||
|
||||
- Keep common hardening (`NoNewPrivileges`, private temp, core/swap limits, system protection, a masked home with narrow read-only runtime/config/unit and PipeWire-socket binds, and socket ownership) in the base service unit. Bind only the selected cloud credential through its generated drop-in. Clear ambient `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY` variables unless proxy support is separately designed and reviewed.
|
||||
- Generate/install an offline drop-in for local mode containing `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`.
|
||||
- Generate/install a cloud drop-in permitting only `AF_UNIX`, `AF_INET`, and `AF_INET6` and no inbound listener beyond the inherited Unix socket. DNS and outbound TLS must be tested under the final unit.
|
||||
- Stamp the expected network mode in the generated unit environment. At startup, compare it with the selected backend and refuse to run on mismatch. This prevents a hand-edited local config from silently running under cloud-capable hardening, and a cloud backend from failing mysteriously under the offline profile.
|
||||
- Restrict service reads to the installed runtime, selected model/dictionary paths, and `~/.config/pi-dictate` where practical.
|
||||
- Document that cloud mode necessarily grants the host transcription process general outbound IP connectivity; systemd cannot safely express a stable hostname allowlist for changing provider addresses.
|
||||
|
||||
Preserve `profiles/nono/pi-local.json` byte-for-byte. The existing resolved policy must be reviewed separately; Pi must not read host config or keys.
|
||||
|
||||
## Implementation phases
|
||||
|
||||
### Phase 0: Freeze provider contracts and run opt-in spikes
|
||||
|
||||
1. Save sanitized request/event fixtures from the current public Meta and OpenAI docs.
|
||||
2. With explicit user consent and test keys, run a short prerecorded, non-sensitive PCM fixture against each API outside Pi.
|
||||
3. Confirm handshake, framing, finalization, close/error behavior, first/final latency, model availability, and documented billing/rate limits.
|
||||
4. Record exact model IDs and API doc dates. Treat a changed or unavailable Meta contract as a planning gate, not a reason to use an undocumented endpoint.
|
||||
|
||||
Completion: one minimal client per provider produces an exact final transcript and no unresolved required schema field remains.
|
||||
|
||||
### Phase 1: Add strict host config and preserve local behavior
|
||||
|
||||
1. Add the versioned config and credential readers with permission, ownership, size, and redaction checks.
|
||||
2. Move local model path/checksum/terms/boost/idle settings from environment assumptions into typed config.
|
||||
3. Introduce backend/turn protocols and wrap the current NeMo path without changing recorder or protocol behavior.
|
||||
4. Add config and local-regression tests.
|
||||
|
||||
Completion: local Dictate passes all existing tests and runs with generated local config under the same offline unit restrictions.
|
||||
|
||||
### Phase 2: Generalize capture for realtime sinks
|
||||
|
||||
1. Make recorder sample rate and byte-duration cap explicit and tested.
|
||||
2. Add a nonblocking PCM sink plus fixed queue/backpressure limits while retaining local-buffer mode.
|
||||
3. Add ordered finish, provider-error polling, cancellation, and bounded worker joins to the service core.
|
||||
4. Test 16/24 kHz commands, exact five-minute limits, stalled consumers, queue overflow, final PCM ordering, cancellation in every phase, and no thread/resource leaks.
|
||||
|
||||
Completion: a fake realtime backend receives byte-for-byte ordered PCM through stop and returns one final result; local behavior remains unchanged.
|
||||
|
||||
### Phase 3: Implement Meta
|
||||
|
||||
1. Implement the public Muse handshake and event parser from Phase 0 fixtures.
|
||||
2. Integrate 24 kHz binary streaming, end-of-input, final-only delivery, timeout, and cancellation.
|
||||
3. Map the coding dictionary into validated Meta bias terms.
|
||||
4. Add fake-server contract tests and an opt-in live smoke test.
|
||||
|
||||
Completion: Meta mode starts recording only after handshake, streams bounded audio, pastes exactly one final transcript, and leaves no deliverable result after cancel/disconnect.
|
||||
|
||||
### Phase 4: Implement OpenAI
|
||||
|
||||
1. Implement authenticated Realtime transcription session setup.
|
||||
2. Integrate base64 appends, explicit commit, item correlation, model/context settings, final-only delivery, timeout, and cancellation.
|
||||
3. Test `gpt-live-transcribe` as default and `gpt-transcribe` as the documented commit-oriented alternative.
|
||||
4. Add fake-server contract tests and an opt-in live smoke test.
|
||||
|
||||
Completion: OpenAI mode satisfies the same user-visible and cleanup contract as Meta and local mode.
|
||||
|
||||
### Phase 5: Make installation and hardening backend-aware
|
||||
|
||||
1. Add locked local/cloud dependency profiles and backend-specific provisioning.
|
||||
2. Add atomic install/configure/check operations and secure key copying.
|
||||
3. Add local/cloud systemd hardening generation plus startup mismatch checks.
|
||||
4. Preserve `profiles/nono/pi-local.json` byte-for-byte and make no Pi policy changes.
|
||||
5. Review the existing resolved profile separately, then test fresh install, every switch direction, reinstall preservation, invalid config, missing/insecure key files, local network denial, cloud DNS/TLS, idle exit, reactivation, and uninstall.
|
||||
|
||||
Completion: selecting a backend installs only its required runtime, applies matching hardening, and reactivates through the same Unix socket after restart.
|
||||
|
||||
### Phase 6: Extension diagnostics and documentation
|
||||
|
||||
1. Use backend-neutral status wording and optionally expose bounded backend/model diagnostics.
|
||||
2. Add `/dictate-status` only if it can remain a simple read-only protocol operation.
|
||||
3. Update `docs/dictate/protocol.md`, `extensions/dictate/README.md`, root `README.md`, attribution, privacy, troubleshooting, and operations sections.
|
||||
4. Document exactly when cloud audio begins leaving the host, credential storage, provider retention/terms links, billing, five-minute cap, no partial editor insertion, and local offline guarantees.
|
||||
|
||||
Completion: docs let a new user install any one backend, switch safely, understand data flow/cost, diagnose failures, and return to local offline mode.
|
||||
|
||||
### Phase 7: Validation and independent review
|
||||
|
||||
1. Run focused TypeScript/Python tests, typecheck, package tests, and the full repository test suite.
|
||||
2. Manually test cold/warm local starts and fresh cloud sessions, stop/cancel/disconnect/shutdown in every state, empty speech, provider auth/rate-limit/outage, network loss, duration limit, two competing Pi clients, service idle exit, and socket reactivation.
|
||||
3. Confirm the local unit cannot resolve/connect to the Internet and the cloud unit cannot expose a TCP listener.
|
||||
4. Confirm Pi's resolved nono policy has no new grants and Pi cannot read Dictate config/keys or reach either provider directly.
|
||||
5. Have an independent reviewer audit credential handling, audio retention, network hardening, provider event validation, cancellation, and installer migration.
|
||||
|
||||
## Test matrix
|
||||
|
||||
| Area | Local | Meta | OpenAI |
|
||||
| --- | --- | --- | --- |
|
||||
| Prepare before recorder | lazy model load | connect/auth/handshake | connect/auth/session ack |
|
||||
| Capture | 16 kHz PCM16 retained | 24 kHz PCM16 queued/binary | 24 kHz PCM16 queued/base64 JSON |
|
||||
| Stop | drain, convert, infer | drain, end marker, final | drain, commit, matching completion |
|
||||
| Context terms | NeMo phrase boost | Meta bias terms | keywords/prompt/languages |
|
||||
| Cancellation | tombstone load/infer | close stream, suppress final | close session, suppress final |
|
||||
| Hardening | AF_UNIX, IP denied | outbound TLS enabled | outbound TLS enabled |
|
||||
| Main result | bounded string | bounded explicit final | bounded matching final |
|
||||
| Live smoke | GPU/model opt-in | key/cost opt-in | key/cost opt-in |
|
||||
|
||||
Every provider also needs tests for malformed/oversized events, stale results, partial-only close, empty final, invalid UTF-8/JSON where applicable, auth failure, rate limit, timeout, queue overflow, recorder failure, service shutdown, and redacted errors.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- One strict host configuration selects local, Meta, or OpenAI and takes effect only after a validated service restart.
|
||||
- Existing users remain local until they explicitly opt into a cloud backend.
|
||||
- The local model path, checksum, terms, and boost are configurable without accepting an unverified archive.
|
||||
- Meta uses only the public `api.meta.ai` Muse Voice Transcribe endpoint and documented bearer contract.
|
||||
- OpenAI defaults to `gpt-live-transcribe`; the model is configurable and `whisper-1` is not silently treated as Realtime-compatible.
|
||||
- Provider connection/session readiness precedes recorder startup.
|
||||
- Cloud PCM is streamed in order through a fixed bounded queue; no audio is silently dropped, retried to another provider, written to disk, or exposed to Pi.
|
||||
- Stop drains the recorder completely before provider finalization.
|
||||
- Only one explicit bounded final result for the active generation can change the editor.
|
||||
- Partial events, cancellation, stale generations, disconnects, provider failures, and cleanup never insert text.
|
||||
- API keys live only in dedicated host credential files, pass strict file checks, never cross the Unix protocol, and never appear in logs, errors, command lines, URLs, redirects, or ambient proxies.
|
||||
- Local mode retains IP denial; cloud mode's broader outbound capability is explicit, tested, and cannot be selected under mismatched hardening.
|
||||
- Cloud-only installation does not provision NeMo/CUDA/model artifacts.
|
||||
- The Pi nono profile gains no new filesystem, socket, network, PipeWire, GPU, or credential capability.
|
||||
- Focused and full tests, typecheck, host smoke tests, and independent security review pass.
|
||||
@@ -29,3 +29,29 @@ The complete transitive dependency versions and artifact hashes are recorded in
|
||||
- NeMo ASR documentation: <https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/asr/intro.html>
|
||||
- CC BY 4.0: <https://creativecommons.org/licenses/by/4.0/>
|
||||
- Apache License 2.0: <https://www.apache.org/licenses/LICENSE-2.0>
|
||||
|
||||
## Optional cloud providers
|
||||
|
||||
Cloud backends are opt-in and are not required for local mode. Pi Dictate does not redistribute Meta or OpenAI models and does not include either provider's SDK. The host service sends audio only after the selected provider session is authenticated and ready. Provider terms, retention, security, and billing policies apply to cloud audio and any keyword or prompt context sent with it.
|
||||
|
||||
### Meta Model API Muse Voice Transcribe
|
||||
|
||||
- Announcement: <https://research.meta.ai/blog/introducing-muse-voice-transcribe>
|
||||
- Speech-to-text guide: <https://ai.developer.meta.com/docs/speech-to-text>
|
||||
- Realtime Voice API: <https://ai.developer.meta.com/docs/api-reference/voice/realtime>
|
||||
- Voice event schemas: <https://ai.developer.meta.com/docs/api-reference/voice/schemas>
|
||||
- Authentication: <https://ai.developer.meta.com/docs/getting-started/authentication>
|
||||
- Meta terms: <https://ai.meta.com/terms/>
|
||||
|
||||
Pi Dictate uses only the documented `api.meta.ai` realtime endpoint and `muse-voice-transcribe-1.0` contract. It does not use the undocumented `shortwave.facebook.com` Muse Code endpoint.
|
||||
|
||||
### OpenAI Realtime transcription
|
||||
|
||||
- Realtime transcription guide: <https://developers.openai.com/api/docs/guides/realtime-transcription>
|
||||
- Realtime WebSocket guide: <https://developers.openai.com/api/docs/guides/realtime-websocket>
|
||||
- Speech-to-text guide: <https://developers.openai.com/api/docs/guides/speech-to-text>
|
||||
- `gpt-live-transcribe` model: <https://developers.openai.com/api/docs/models/gpt-live-transcribe>
|
||||
- OpenAI terms: <https://openai.com/policies/terms-of-use/>
|
||||
- OpenAI privacy policy: <https://openai.com/policies/privacy-policy/>
|
||||
|
||||
The OpenAI default is `gpt-live-transcribe`; `gpt-transcribe` is a configurable commit-oriented alternative. `whisper-1` is not treated as a Realtime model.
|
||||
|
||||
@@ -46,3 +46,18 @@ A 37-term dictionary and boost strengths from 0.25 through 2.0 were tested again
|
||||
The packaging, latency, memory, offline, and normal-dictation gates pass. Alpha `1.0` produced the same expected clean transcript as baseline, the same degraded transcript on seeded noisy input, and an empty transcript for silence. No dictionary hallucination was observed.
|
||||
|
||||
The coding-term remediation passes the final focused live check through the installed socket-activated service. The spoken phrase “Run npm test, then git status.” produced exactly `Run npm test, then git status.` with native phrase boosting at alpha `1.0`.
|
||||
|
||||
The upgraded configurable service was revalidated on 03SEP2026. Immediate cancellation during preparation was delivery-neutral, a second generation started successfully, and a six-second end-to-end capture returned `Run npm test, then get status.` The wording variation does not replace the earlier accuracy fixture; it confirms the current installer, systemd sandbox, socket protocol, PipeWire capture, in-memory model path, and final-only delivery operate together. Local systemd policy exposed only the PipeWire runtime socket while retaining the masked home and IP-network denial.
|
||||
|
||||
## Configurable backend contract gate
|
||||
|
||||
Credentialed live contract checks passed on 03SEP2026 with a five-second, non-sensitive 24 kHz mono PCM microphone sample held only in memory. Dedicated mode-`0600` provider key files were read by the host-side adapters; no credential entered Pi, a command line, a URL, or captured output.
|
||||
|
||||
| Provider | Model | Result |
|
||||
| --- | --- | --- |
|
||||
| Meta | `muse-voice-transcribe-1.0` | The public `api.meta.ai` handshake accepted the session, streamed PCM, accepted `endStream`, and returned an explicit final transcript (`run npm test`). Total turn time was 30.5 seconds. |
|
||||
| OpenAI | `gpt-live-transcribe` | The GA transcription session accepted 24 kHz PCM, explicit commit, and returned a correlated completion event (`Run npm test and`). Total turn time was 7.1 seconds. |
|
||||
|
||||
The first OpenAI spike exposed forward-compatible fields (`event_id` and `previous_item_id`) on the commit acknowledgement. The adapter and sanitized fixture now accept those bounded fields while still requiring a non-empty `item_id`. A separate credentialed session update confirmed that `gpt-live-transcribe` accepts transcription `delay: "low"`. Configuration therefore supports only the documented bounded values `minimal`, `low`, `medium`, `high`, and `xhigh`; omission keeps the provider default.
|
||||
|
||||
These checks validate the current wire contract and finalization path, not comparative transcription accuracy. The public contract references checked were the [Meta Realtime Voice API](https://ai.developer.meta.com/docs/api-reference/voice/realtime), [Meta event schemas](https://ai.developer.meta.com/docs/api-reference/voice/schemas), [OpenAI Realtime transcription guide](https://developers.openai.com/api/docs/guides/realtime-transcription), [OpenAI Realtime WebSocket guide](https://developers.openai.com/api/docs/guides/realtime-websocket), and [`gpt-live-transcribe` model page](https://developers.openai.com/api/docs/models/gpt-live-transcribe).
|
||||
|
||||
+72
-26
@@ -1,28 +1,34 @@
|
||||
# Pi Dictate local-service protocol
|
||||
# Pi Dictate host-service protocol
|
||||
|
||||
## Transport and version
|
||||
|
||||
Protocol version `2` uses UTF-8, newline-delimited JSON over one persistent Unix stream socket. Each line contains one JSON object. The systemd socket unit owns the socket path and permissions. The service accepts the inherited listening file descriptor and does not bind or unlink the path.
|
||||
|
||||
Clients must send `hello` before any other request. A connection negotiates one version only. Coding terms are host-owned service configuration and never cross this protocol.
|
||||
Clients must send `hello` before any other request. A connection negotiates one version only. The selected backend, provider credentials, coding dictionary, model paths, and provider wire events are host-service configuration. They never cross this protocol.
|
||||
|
||||
## Limits
|
||||
The protocol is backend-neutral. The host service owns microphone capture and the selected transcription backend. In local mode it runs the pinned NeMo/Parakeet adapter offline. In Meta and OpenAI modes it owns the provider WebSocket, key, PCM stream, partial hypotheses, and finalization. The Pi extension sees only state, level, result, error, cancellation, and bounded diagnostics.
|
||||
|
||||
## Limits and validation
|
||||
|
||||
| Data | Limit |
|
||||
| --- | ---: |
|
||||
| JSON frame, excluding newline | 65,536 bytes |
|
||||
| `requestId` or `generationId` | 128 UTF-8 bytes |
|
||||
| `backend` or `model` diagnostic | 128 UTF-8 bytes |
|
||||
| Final transcript | 32,768 UTF-8 bytes |
|
||||
| Error detail | 2,048 UTF-8 bytes |
|
||||
| Simultaneous client connections | 16 |
|
||||
| Pending output per client | 65,536 bytes |
|
||||
| Recorder stderr tail | 65,536 bytes |
|
||||
| Audio-level event rate | 10 per second |
|
||||
| Mono 16 kHz signed-16 PCM | 9,600,000 bytes, five minutes |
|
||||
| Local retained PCM | 9,600,000 bytes, five minutes at 16 kHz |
|
||||
| Cloud queued PCM | Fixed and bounded; the whole recording is never retained |
|
||||
|
||||
IDs are non-empty, opaque strings generated by the client. The service compares them only for request correlation, ownership, and stale-generation rejection. It does not interpret IDs as paths. Request objects reject unknown or missing fields.
|
||||
|
||||
Malformed JSON, a frame over the limit, or an unsupported version produces one bounded `error` frame and then closes the connection. The malformed-frame error uses an empty `requestId` when no client ID can be recovered. Schema, state, busy, and ownership errors leave the connection open.
|
||||
`backend` and `model` are optional diagnostic strings on `pong`. When present, they are non-empty, bounded, and safe for one-line display. They identify only the selected backend and model. They must not contain paths, query strings, key material, credential state, provider payloads, or endpoint details. Older services that omit these fields remain protocol-compatible; clients display `unknown` rather than inferring a backend.
|
||||
|
||||
Malformed JSON, invalid UTF-8, a frame over the limit, or an unsupported version produces one bounded `error` frame and then closes the connection. The malformed-frame error uses an empty `requestId` when no client ID can be recovered. Schema, state, busy, and ownership errors leave the connection open.
|
||||
|
||||
## Client requests
|
||||
|
||||
@@ -36,7 +42,7 @@ All examples are one line when transmitted.
|
||||
|
||||
Exact fields: `version`, `type`, `requestId`.
|
||||
|
||||
A successful response is `pong` with the same `requestId`. `ready` reports whether the model is warm.
|
||||
A successful response is `pong` with the same `requestId`. The response may include `service`, `ready`, `busy`, `backend`, and `model`. `ready` says whether the selected backend can begin immediately without preparation. The extension does not use `ready` for state transitions because a cold local model or a new cloud session may need preparation after a successful hello.
|
||||
|
||||
### `ping`
|
||||
|
||||
@@ -44,7 +50,9 @@ A successful response is `pong` with the same `requestId`. `ready` reports wheth
|
||||
{"version":2,"type":"ping","requestId":"ping-1"}
|
||||
```
|
||||
|
||||
Exact fields: `version`, `type`, `requestId`. `hello` is required first. The response is `pong` and may include `ready` and `busy`.
|
||||
Exact fields: `version`, `type`, `requestId`. `hello` is required first. The response is `pong` and may include `ready`, `busy`, `backend`, and `model`.
|
||||
|
||||
The `/dictate-status` extension command uses a short-lived connection: it sends `hello`, waits for its `pong`, sends `ping`, reports the bounded backend/model fields from the ping response, and closes the connection. It never starts a generation or reveals configuration and credential state.
|
||||
|
||||
### `start`
|
||||
|
||||
@@ -52,7 +60,13 @@ Exact fields: `version`, `type`, `requestId`. `hello` is required first. The res
|
||||
{"version":2,"type":"start","requestId":"start-1","generationId":"generation-1"}
|
||||
```
|
||||
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. The service accepts only one global generation. It first emits `state: loading` for a cold model or `state: listening` for a warm model. Recording starts only after model loading succeeds.
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. The service accepts only one global generation. `start` prepares the selected backend before capture:
|
||||
|
||||
- local: load or reuse the checksum-verified NeMo/Parakeet model;
|
||||
- Meta: connect to the fixed public endpoint, authenticate with the handshake frame, and wait for the session acknowledgement;
|
||||
- OpenAI: connect to the fixed Realtime transcription endpoint, wait for `session.created`, send session configuration, and wait for `session.updated`.
|
||||
|
||||
Recording begins only after preparation succeeds. The service declares the recorder rate internally: 16 kHz for local mode and 24 kHz for either cloud mode. A cold start emits `state: loading` while the extension displays `preparing transcription...`; it never displays `listening` before the recorder and selected backend are ready.
|
||||
|
||||
### `stop`
|
||||
|
||||
@@ -60,7 +74,7 @@ Exact fields: `version`, `type`, `requestId`, `generationId`. The service accept
|
||||
{"version":2,"type":"stop","requestId":"stop-1","generationId":"generation-1"}
|
||||
```
|
||||
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. Only the owning connection may stop the active listening generation.
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. Only the owning connection may stop the active listening generation. The service marks the generation stopping, signals `pw-record`, drains stdout to EOF, waits for child close, and then finalizes the selected backend. Local mode converts the complete PCM buffer and runs one inference. Cloud mode sends the provider end/commit marker only after every recorder chunk has been enqueued, then waits for one explicit provider final event.
|
||||
|
||||
### `cancel`
|
||||
|
||||
@@ -68,7 +82,7 @@ Exact fields: `version`, `type`, `requestId`, `generationId`. Only the owning co
|
||||
{"version":2,"type":"cancel","requestId":"cancel-1","generationId":"generation-1"}
|
||||
```
|
||||
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. A valid cancellation immediately emits `cancelled`. Cancellation tombstones the generation, so it can never emit a later `result` or begin recording after a cancelled load.
|
||||
Exact fields: `version`, `type`, `requestId`, `generationId`. A valid cancellation immediately emits `cancelled`. Cancellation tombstones the generation, so it can never later emit a `result` or begin recording after cancelled preparation. The host closes a cloud provider turn, stops capture, joins workers with bounded timeouts, and discards retained or queued PCM as applicable.
|
||||
|
||||
## Service events
|
||||
|
||||
@@ -77,9 +91,11 @@ All service events contain `version`, `type`, and a bounded client-generated `re
|
||||
### `pong`
|
||||
|
||||
```json
|
||||
{"version":2,"type":"pong","requestId":"hello-1","service":"pi-dictate","ready":false}
|
||||
{"version":2,"type":"pong","requestId":"hello-1","service":"pi-dictate","ready":false,"busy":false,"backend":"local","model":"nemo-parakeet"}
|
||||
```
|
||||
|
||||
`service` is the fixed service identifier. `ready` and `busy` are bounded booleans. `backend` and `model` are optional bounded diagnostics. No event reports key presence, file paths, provider URLs, network state, or raw provider responses.
|
||||
|
||||
### `state`
|
||||
|
||||
```json
|
||||
@@ -88,13 +104,15 @@ All service events contain `version`, `type`, and a bounded client-generated `re
|
||||
|
||||
`state` is one of `loading`, `listening`, `stopping`, `transcribing`, or `idle`.
|
||||
|
||||
The wire name `loading` is retained for compatibility. It means backend preparation and includes local model load, cloud connection/authentication, or session setup. The extension renders it as `preparing transcription...`, not `loading model...`.
|
||||
|
||||
### `level`
|
||||
|
||||
```json
|
||||
{"version":2,"type":"level","requestId":"start-1","generationId":"generation-1","level":0.625}
|
||||
```
|
||||
|
||||
While listening, the service emits at most ten `level` events per second. `level` is a finite number from `0` through `1`, derived from the recorder's current 100 ms RMS window on a logarithmic -60 dBFS to 0 dBFS scale. The event carries no PCM samples and stops before recorder shutdown begins.
|
||||
While listening, the service emits at most ten `level` events per second. `level` is a finite number from `0` through `1`, derived from the recorder's current 100 ms RMS window on a logarithmic -60 dBFS to 0 dBFS scale. The event carries no PCM samples and stops before recorder shutdown begins. The recorder rate is selected by the host backend and is not exposed in the event.
|
||||
|
||||
### `result`
|
||||
|
||||
@@ -102,7 +120,7 @@ While listening, the service emits at most ten `level` events per second. `level
|
||||
{"version":2,"type":"result","requestId":"stop-1","generationId":"generation-1","transcript":"example text"}
|
||||
```
|
||||
|
||||
There is at most one result for a generation. An empty transcript is a successful result with `"transcript":""`.
|
||||
There is at most one result for a generation. An empty transcript is a successful result with `"transcript":""`. Only an explicit final provider event may produce this event. Provider partials, cumulative hypotheses, endpointing, diarization, stale completions, or cleanup never produce a result.
|
||||
|
||||
### `cancelled`
|
||||
|
||||
@@ -110,7 +128,7 @@ There is at most one result for a generation. An empty transcript is a successfu
|
||||
{"version":2,"type":"cancelled","requestId":"cancel-1","generationId":"generation-1"}
|
||||
```
|
||||
|
||||
This is a terminal delivery guarantee: the generation cannot later produce a result.
|
||||
This is a terminal delivery guarantee: the generation cannot later produce a result. The service may remain busy while a non-interruptible local inference or provider cleanup worker finishes.
|
||||
|
||||
### `error`
|
||||
|
||||
@@ -123,37 +141,65 @@ Stable codes include:
|
||||
- framing and schema: `malformed_json`, `frame_too_large`, `unsupported_version`, `invalid_request`;
|
||||
- session: `hello_required`, `busy`, `not_owner`, `generation_mismatch`, `invalid_state`;
|
||||
- recorder and audio: `recorder_spawn`, `recorder_exit`, `recorder_timeout`, `duration_limit`, `invalid_audio`;
|
||||
- model and output: `model_failure`, `transcript_too_large`.
|
||||
- local preparation and output: `config_invalid`, `credentials_invalid`, `model_failure`, `transcript_too_large`;
|
||||
- Meta provider failures: bounded `meta_auth`, `meta_rate_limit`, `meta_timeout`, `meta_protocol`, `meta_premature_close`, or another stable `meta_*` code;
|
||||
- OpenAI provider failures: bounded `openai_auth`, `openai_rate_limit`, `openai_timeout`, `openai_protocol`, `openai_premature_close`, or another stable `openai_*` code.
|
||||
|
||||
Error details are bounded and redacted. They never include API keys, authorization headers, credential paths, request bodies, raw provider payloads, or raw audio. A provider's malformed, oversized, invalid-UTF-8, partial-only, premature-close, authentication, rate-limit, duration, and network failures are terminal for the active generation. The service discards the generation rather than returning a plausible incomplete transcript.
|
||||
|
||||
## Provider finalization contract
|
||||
|
||||
The extension does not receive provider frames, but the host service must enforce these rules before emitting a protocol result.
|
||||
|
||||
### Meta Model API Muse Voice Transcribe
|
||||
|
||||
The service uses `wss://api.meta.ai/v1/asr/realtime` with the documented first-frame bearer handshake for `muse-voice-transcribe-1.0`, `PCM_24KHZ`, `PUSH_TO_TALK`, and cumulative partials. It waits for the `sessionId` acknowledgement before capture, sends ordered binary PCM16 frames through a bounded queue, sends `{ "type": "endStream" }` after recorder EOF, and requires a `transcript` event with `final: true`. It consumes the normal `1000` close and maps close codes `1008`, `1011`, and `1013` to stable failures.
|
||||
|
||||
The service must use only the public endpoint. It must not use the undocumented `shortwave.facebook.com` Muse Code endpoint. See the [Meta API references](https://ai.developer.meta.com/docs/api-reference/voice/realtime), [event schemas](https://ai.developer.meta.com/docs/api-reference/voice/schemas), and [authentication guide](https://ai.developer.meta.com/docs/getting-started/authentication).
|
||||
|
||||
### OpenAI Realtime transcription
|
||||
|
||||
The service uses `wss://api.openai.com/v1/realtime?intent=transcription` with `Authorization: Bearer <key>`. It waits for `session.created`, sends the GA nested `session.update` with `type: "transcription"`, the configured model, 24 kHz mono PCM16, and `turn_detection: null`, then waits for `session.updated`. Audio uses bounded base64 `input_audio_buffer.append` events. Stop commits only after recorder EOF and correlates `conversation.item.input_audio_transcription.completed` to the committed item. Delta and unmatched events are ignored.
|
||||
|
||||
The default is `gpt-live-transcribe`. `gpt-transcribe` is the documented commit-oriented alternative. `whisper-1` is not silently used as a Realtime model. A credentialed live check on 03SEP2026 confirmed that `gpt-live-transcribe` accepts transcription `delay: "low"`; configuration permits `minimal`, `low`, `medium`, `high`, and `xhigh`, while omission keeps the provider default. See the [Realtime transcription guide](https://developers.openai.com/api/docs/guides/realtime-transcription), [WebSocket guide](https://developers.openai.com/api/docs/guides/realtime-websocket), and [`gpt-live-transcribe` model page](https://developers.openai.com/api/docs/models/gpt-live-transcribe).
|
||||
|
||||
## Lifecycle and ownership
|
||||
|
||||
| Current state | Request or event | Outcome |
|
||||
| --- | --- | --- |
|
||||
| no generation | owner `start` | `loading`, then `listening`; warm models may emit `listening` directly |
|
||||
| no generation | owner `start` | `loading`, then `listening`; a prepared backend may emit `listening` directly |
|
||||
| any active generation | another `start` | `busy`; connection stays open |
|
||||
| `loading` | owner `cancel` | immediate `cancelled`; loading finishes in the worker but recording is suppressed |
|
||||
| `loading` | owner `cancel` | immediate `cancelled`; preparation finishes in the worker but recording is suppressed |
|
||||
| `listening` | recorder audio | rate-limited `level` events for the owning client |
|
||||
| `listening` | owner `stop` | `stopping`; signal recorder, drain stdout to EOF, await child close, convert PCM, then `transcribing` |
|
||||
| `listening` | owner `cancel` or disconnect | stop and drain recorder, discard audio, never infer |
|
||||
| `listening` | owner `stop` | `stopping`; stop and drain recorder fully, then `transcribing` and one local/provider final result |
|
||||
| `listening` | owner `cancel` or disconnect | stop and drain recorder, discard audio, never infer or finalize |
|
||||
| `stopping` | owner `cancel` | immediate `cancelled`; completed audio is discarded |
|
||||
| `transcribing` | owner `cancel` or disconnect | tombstone immediately; inference finishes but its result is discarded |
|
||||
| `transcribing` | successful model result | one `result`, then `idle` |
|
||||
| `transcribing` | owner `cancel` or disconnect | tombstone immediately; local inference/provider finalization finishes but its result is discarded |
|
||||
| `transcribing` | successful explicit final | one `result`, then `idle` |
|
||||
| any work state | terminal failure | one `error`, then `idle` |
|
||||
| active generation | non-owner `stop` or `cancel` | `not_owner`; connection stays open |
|
||||
| active generation | stale generation ID | `generation_mismatch`; connection stays open |
|
||||
|
||||
Cancelled model loading and inference are non-interruptible. The global service remains busy until the worker finishes even though `cancelled` was already emitted. Disconnecting a generation owner cancels capture or tombstones loading/inference under the same rules.
|
||||
The service is globally single-generation. A cloud queue overflow, provider stall, or worker failure is terminal. PCM is never silently dropped, retried to another provider, written to disk, or exposed to Pi.
|
||||
|
||||
The recorder command is exactly:
|
||||
The recorder command is selected by the host service and is otherwise fixed:
|
||||
|
||||
```text
|
||||
pw-record --raw --format s16 --rate 16000 --channels 1 -
|
||||
```
|
||||
|
||||
The service launches `pw-record` in its own process group and marks the generation stopping before SIGTERM. It continuously drains bounded stdout and stderr, waits a bounded time for pipe EOF and child close, and only then converts or transcribes. A shutdown timeout escalates to a process-group kill and fails the generation. PipeWire's exit code `1` is accepted only after this service intentionally signalled a still-running recorder and stderr remained empty; an already-exited process or stderr-bearing exit remains a failure. Raw audio and recorder logs are held only in bounded memory and are not written to disk.
|
||||
The local adapter uses 16 kHz. Meta and OpenAI use 24 kHz. The service launches `pw-record` in its own process group, marks the generation stopping before SIGTERM, continuously drains bounded stdout and stderr, waits for pipe EOF and child close, and only then converts or finalizes. A shutdown timeout escalates to a process-group kill and fails the generation. The service creates no audio file.
|
||||
|
||||
Cancellation and disconnect are delivery tombstones. They stop capture, close a provider turn where present, release application-owned buffers, and suppress every later final event. Cleanup is best effort and cannot authorize editor delivery.
|
||||
|
||||
## Idle exit
|
||||
|
||||
A successfully parsed protocol request resets the 15-minute monotonic idle activity timer, including requests that later produce state or ownership errors. Accepting a socket, receiving malformed data, and disconnecting do not reset activity.
|
||||
A successfully parsed protocol request resets the configured monotonic idle activity timer, including requests that later produce state or ownership errors. Accepting a socket, receiving malformed data, and disconnecting do not reset activity.
|
||||
|
||||
Idle exit is suspended while model loading, recording, recorder shutdown, PCM conversion, or inference is active. Once work becomes idle, the full timer starts again. Exiting closes accepted clients and the inherited listener but does not unlink the systemd-owned socket path.
|
||||
Idle exit is suspended while backend preparation, recording, recorder shutdown, PCM conversion, local inference, cloud streaming, provider finalization, or worker cleanup is active. Once work becomes idle, the configured timer starts again. It is 15 minutes by default. Exiting closes accepted clients and the inherited listener but does not unlink the systemd-owned socket path. The service's single owned worker is daemonized so bounded core shutdown permits normal interpreter exit even if native code is stuck. Explicit systemd shutdown also has a five-second stop deadline.
|
||||
|
||||
## Configuration and security boundary
|
||||
|
||||
The service reads strict versioned host configuration at startup from `~/.config/pi-dictate/config.json` and reads only the selected backend's regular, mode-`0600`, service-owned credential file. Keys never appear in command lines, systemd environment variables, URLs, redirects, protocol frames, diagnostics, or logs. Pi has no configuration writer and no access to this directory.
|
||||
|
||||
Local hardening denies IP networking and keeps the pinned runtime offline. Cloud hardening permits only `AF_UNIX`, `AF_INET`, and `AF_INET6` needed for provider DNS/TLS, while the base unit denies `bind()` and `listen()` so the service cannot create an inbound TCP listener. The service rejects a selected backend when the stamped network mode does not match the installed unit. Ambient `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY` variables are cleared unless separately reviewed. The exact host install and switch operations are documented in [`extensions/dictate/README.md`](../../extensions/dictate/README.md).
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Dictate extension review and local Parakeet remediation plan
|
||||
|
||||
Status: implemented, with final host validation in progress. This document records the original review of `extensions/dictate/index.ts`, the agreed Linux implementation direction, and the acceptance criteria used for the replacement.
|
||||
Status: historical local-backend review; final host validation completed. This document records the original review of `extensions/dictate/index.ts`, the agreed Linux implementation direction, and the acceptance criteria used for the local replacement. The current configurable local, Meta, and OpenAI scope supersedes its local-only decisions; see [`configurable-asr-backends-plan.md`](configurable-asr-backends-plan.md) and [`protocol.md`](protocol.md).
|
||||
|
||||
## Decisions
|
||||
|
||||
The replacement will target this machine and workflow:
|
||||
The replacement will target this machine and workflow. The checked-in `profiles/nono/pi-local.json` is outside this change and remains byte-for-byte unchanged.
|
||||
|
||||
- **Platform:** Linux only, specifically CachyOS on x86-64 with niri and Ghostty.
|
||||
- **Transcription:** local NVIDIA NeMo with `nvidia/parakeet-tdt-0.6b-v3` on the machine's RTX 4090. Deepgram and runtime cloud transcription are removed.
|
||||
@@ -172,7 +172,7 @@ The service validates even-byte PCM length, converts little-endian signed 16-bit
|
||||
|
||||
Running capture and NeMo inside Pi would require PipeWire, model-directory, native-library, CUDA, and `/dev/nvidia*` access in the agent sandbox. Deepgram also requires cloud credentials and a network destination.
|
||||
|
||||
**Fix:** keep direct device and model capabilities in the host-side service. Add only `/run/user/1000/pi-dictate.sock` to `filesystem.unix_socket` in `profiles/nono/pi-local.json`. Do not add PipeWire configuration, `/run/user/1000/pipewire-0`, `/dev/snd`, `/dev/nvidia*`, model paths, Hugging Face cache paths, or `api.deepgram.com`.
|
||||
**Fix:** keep direct device and model capabilities in the host-side service. Preserve the existing `/run/user/1000/pi-dictate.sock` policy entry and make no profile edits. Do not add PipeWire configuration, `/run/user/1000/pipewire-0`, `/dev/snd`, `/dev/nvidia*`, model paths, Hugging Face cache paths, or `api.deepgram.com`.
|
||||
|
||||
The socket is deliberately a mediated microphone and transcription capability, not a harmless health endpoint. Any process running under `pi-local` or a descendant profile can request recording and receive the resulting text. The user accepts that broad profile scope; document it prominently and keep the service single-user with socket mode `0600`.
|
||||
|
||||
@@ -375,10 +375,10 @@ Build a throwaway prototype outside the production extension and nono sandbox:
|
||||
|
||||
### Phase 4: Add the minimum nono capability
|
||||
|
||||
1. Add `/run/user/1000/pi-dictate.sock` to `filesystem.unix_socket` in `profiles/nono/pi-local.json`.
|
||||
1. Preserve the existing `/run/user/1000/pi-dictate.sock` policy entry and make no profile edits.
|
||||
2. Do not add PipeWire, GPU, model, cache, Deepgram, or extra network permissions.
|
||||
3. Validate the draft and resolved policy according to the nono-profile workflow.
|
||||
4. Compare the resolved policy before and after and confirm this feature adds only the exact service socket.
|
||||
3. Validate the existing resolved policy separately.
|
||||
4. Confirm this feature does not change the resolved Pi policy.
|
||||
5. Restart Pi because an active nono sandbox cannot gain socket access in place.
|
||||
6. Verify the extension can activate and use the service but still cannot connect directly to PipeWire or initialize NVML.
|
||||
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
# Fast-browser development contracts
|
||||
|
||||
Status: frozen implementation boundary, 2026-09-19. The TypeScript source of truth is
|
||||
`extensions/fast-browser/contracts.ts`; this document explains ownership and safety
|
||||
rules, not a second protocol.
|
||||
|
||||
## Published exports
|
||||
|
||||
`BrowserBridge` remains the prototype observe/act/close contract. New work must
|
||||
implement `BrowserRuntime`, a persistent, session-owned extension of that
|
||||
contract. The published operation names are:
|
||||
|
||||
`observe`, `act`, `read`, `assert`, `navigate`, `reload`, `listTabs`,
|
||||
`selectTab`, `waitPopup`, `closeTab`, `captureEvidence`, `narrowOriginPolicy`,
|
||||
and `close`.
|
||||
|
||||
`EvidenceRequest.operation` uses the exported
|
||||
`BROWSER_EVIDENCE_OPERATIONS`/`BrowserEvidenceOperationName` allowlist. It
|
||||
contains every supported capture label except session `close`; `closeTab`
|
||||
remains a supported distinct label.
|
||||
|
||||
`BrowserOperation` is the explicit JSONL/CLI request union. `ScenarioOperation`
|
||||
wraps the existing bounded `Step` as `{op:"step",step}` and reuses the fixed
|
||||
read/assert/control request arms. It has no JavaScript, selector, raw CDP, or
|
||||
arbitrary command arm. `FAST_BROWSER_SCENARIO_FILE_VERSION` and
|
||||
`FAST_BROWSER_REPORT_VERSION` are both `1`; scenario owners may add their
|
||||
validated setup/reset/cleanup details around this union. Scenario input is an
|
||||
explicit caller-provided file path; it is not restricted to fixture IDs.
|
||||
|
||||
Assertions are finite and read-only: URL `exact` or `origin`, title `exact` or
|
||||
`contains`, and visible-text `contains`. `wait:"bounded"` uses the fixed
|
||||
3-second read/assertion budget; `wait:"none"` performs one read. A failed or
|
||||
stale read is an operation error, never a mutation or retry. `read` returns only
|
||||
bounded rendered visible text, title, and URL metadata; it excludes input/value
|
||||
properties, password values, hidden text, HTML, frames, and script output.
|
||||
|
||||
Tab IDs are opaque runtime-issued handles. A handle remains valid across calls
|
||||
for the same target; `documentId` changes on navigation/reload. `listTabs` is
|
||||
origin-filtered and deterministic. `waitPopup` returns an owned popup handle
|
||||
with `autoSwitched:false`; only `selectTab` changes the selected target. No
|
||||
operation infers a target from page text.
|
||||
|
||||
## Scenario-only tab references
|
||||
|
||||
Runtime and transport calls still use opaque `BrowserTabId` values. JSON scenario
|
||||
files use the bounded, data-only `ScenarioTabReference` instead; the CLI resolves
|
||||
it to an opaque ID at execution time. References are either the initial selected
|
||||
tab, an earlier operation's `selectedTab`/`popup` result, or an earlier
|
||||
`listTabs` result at `ScenarioTabIndex` 0–31. Operation references may use an
|
||||
explicit prior numeric `index` (0–31) or prior unique `id`; forward, missing,
|
||||
duplicate, and out-of-range references are rejected before execution.
|
||||
|
||||
A list operation may include an exact-origin or bounded-index `select` predicate.
|
||||
An origin predicate must match exactly one listed tab. Listing never changes the
|
||||
runtime selection; a later explicit `selectTab` operation does that. Popup waits
|
||||
also never auto-switch.
|
||||
|
||||
Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"id": "popup-check",
|
||||
"operations": [
|
||||
{"id":"start","op":"read","request":{"tab":{"source":"initial","target":"selected"}}},
|
||||
{"id":"tabs","op":"listTabs","request":{"select":{"kind":"origin","origin":"http://127.0.0.1:4310"}}},
|
||||
{"id":"selected","op":"selectTab","request":{"tab":{"source":"operation","operation":{"id":"tabs"},"result":"listedTab","index":0}}},
|
||||
{"id":"popup","op":"waitPopup","request":{"opener":{"source":"operation","operation":{"id":"selected"},"result":"selectedTab"}}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`selectedTab` is produced by `selectTab`, `popup` by `waitPopup`, and `listedTab`
|
||||
by `listTabs`; there is no interpolation, templating, selector, or script arm.
|
||||
|
||||
## Scenario fixture/server lifecycle
|
||||
|
||||
`ScenarioDefinition.fixture` is optional. When present, `baseUrl` is an explicit
|
||||
credential-free HTTP(S) URL and `server` is either:
|
||||
|
||||
- `{"mode":"managed","start":...,"readiness":...}`: the CLI starts one
|
||||
long-lived child with trusted argv (`shell:false`), optional cwd, and existing
|
||||
env-reference shape, then owns that process until the scenario `finally` path;
|
||||
- `{"mode":"external","readiness":...}`: no child is started. The caller
|
||||
supplies the explicit already-running readiness URL.
|
||||
|
||||
`ScenarioHttpReadiness` is GET-only and contains only `url`, optional expected
|
||||
status, and a bounded timeout. It has no headers, body, shell, or page-derived
|
||||
command fields. Setup/reset/cleanup hooks remain one-shot hooks and are distinct
|
||||
from the managed server child. The CLI's existing scenario-wide
|
||||
`budgets.timeoutMs` starts before any hook or server startup; no second global
|
||||
deadline knob is added. The runner polls readiness within the published
|
||||
`FAST_BROWSER_SCENARIO_SERVER_BOUNDS`, keeps the managed child owned through all
|
||||
operations, and performs bounded shutdown in `finally`. Existing definitions
|
||||
without a managed server remain valid when the CLI/user config provides an
|
||||
explicit external readiness URL.
|
||||
|
||||
Example execution order:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"id": "fixture-login",
|
||||
"fixture": {
|
||||
"baseUrl": "http://127.0.0.1:4310",
|
||||
"server": {
|
||||
"mode": "managed",
|
||||
"start": {"command":"npm","args":["run","fixture"]},
|
||||
"readiness": {"url":"http://127.0.0.1:4310/health","expectedStatus":200}
|
||||
}
|
||||
},
|
||||
"operations": [{"op":"step","step":{"instruction":"Continue","action":"click","postcondition":{"kind":"text","value":"Done"}}}]
|
||||
}
|
||||
```
|
||||
|
||||
Order is: start the existing global scenario budget → setup hook → managed
|
||||
server start and bounded GET readiness (or external readiness) → reset hook →
|
||||
browser operations → finally close browser/runtime, run cleanup while the owned
|
||||
server is still available, bounded server stop, and close uncommitted artifact
|
||||
resources while retaining committed artifacts. A readiness failure stops before
|
||||
browser operations and never becomes a page-derived command.
|
||||
|
||||
## Policy, budgets, and errors
|
||||
|
||||
`RuntimeOriginPolicy` preserves the current `"*"` default or a non-empty exact
|
||||
origin list. `narrowOriginPolicy` computes a `RuntimeOperationContext` for an
|
||||
effective request, constrained by the immutable session base; it never
|
||||
irreversibly tightens later calls. Pass that context to the scoped calls, or
|
||||
omit it on a later call to restore the session base. A restricted base never
|
||||
widens. Navigation accepts only bounded HTTP(S) URLs without credentials whose
|
||||
origin is approved. A runtime owns one session's CDP/worker state and
|
||||
serializes its calls; distinct sessions may run in parallel. Its lifetime is
|
||||
the browser-session lifetime, not the first tool call. Cancellation must stop
|
||||
the worker/daemon and release no resource until teardown is confirmed. Close is
|
||||
idempotent and best effort.
|
||||
|
||||
`FAST_BROWSER_BUDGETS` publishes defaults and hard ceilings for startup (10s),
|
||||
worker readiness (5s), bridge request (10s/120s ceiling for the existing
|
||||
compatibility seam), action verification (3s), read/assertion wait (3s),
|
||||
navigation/reload (10s), popup wait (5s), screenshot (10s/60s ceiling), failure
|
||||
evidence (2s), and a run (60s/120s). Do not add a generic unbounded timeout
|
||||
options object. Failure capture is bounded and best effort; it must never mask
|
||||
the original operation result or retry a mutation.
|
||||
|
||||
All safe failures use `BrowserErrorDetails`/`BrowserOperationError`, with an
|
||||
operation, phase, safe message, error code, retry classification, and mandatory
|
||||
`mutationState` (`not_attempted`, `verified`, or `unknown`). Raw exceptions are
|
||||
not reportable.
|
||||
|
||||
## Transport boundary
|
||||
|
||||
`BrowserTransport` is the typed JSONL boundary. Its exact API is
|
||||
`dispatch<Operation extends BrowserOperation>(request: BrowserTransportRequest<Operation>,
|
||||
signal): Promise<BrowserTransportResultFor<Operation>>` and
|
||||
`close(): Promise<void>`. `BrowserTransportRequest` contains one
|
||||
`BrowserOperation` plus an optional scoped `RuntimeOperationContext`;
|
||||
`BrowserTransportResultFor<Operation>` is the exact result mapping for every
|
||||
`BrowserOperation` variant, and `BrowserTransportResult` is its complete union.
|
||||
Evidence transport uses `BrowserTransportArtifactPayload`: JSONL
|
||||
`encoding:"base64"`, `dataBase64`, declared `byteLength`, content type, kind,
|
||||
and truncation flag. The runtime decodes those bytes and writes them to the
|
||||
parent `PrivateArtifactStore`; no caller path crosses into the worker.
|
||||
`FAST_BROWSER_ARTIFACT_LIMITS` caps screenshots at 8 MiB and diagnostics at
|
||||
256 KiB. `BrowserTransportFactory.create({cdpUrl, allowedOrigins, python?},
|
||||
parentSignal?)` owns worker framing, validation, deadlines, and teardown.
|
||||
`BrowserRuntime` owns persistent tab/policy state and delegates to this
|
||||
transport; these layers must not both implement the JSONL protocol.
|
||||
|
||||
`BrowserRuntimeOptions.python` and `BrowserTransportOptions.python` are the
|
||||
explicit interpreter-selection seam for the pinned environment. If omitted,
|
||||
the existing configured/default bridge resolution remains responsible for
|
||||
choosing Python; no worker may silently drop to a user-global environment when
|
||||
an explicit interpreter/factory is supplied.
|
||||
|
||||
## Evidence and artifact ownership
|
||||
|
||||
`PrivateArtifactStore.open(owner, kind)` is the only write API. Evidence
|
||||
requests carry the caller/scenario `runId`; the runtime creates the operation
|
||||
identity. There is no caller-selected path. The session-owned store creates
|
||||
private directories and atomically commits `0600` files under its private root,
|
||||
conceptually `<root>/<sessionId>/<runId>/<artifactId>.(png|json)`. The returned
|
||||
`PrivateArtifactMetadata` records owner, path, byte count, content type, and
|
||||
`localOnly:true/modelForwarded:false`. Runtime close closes the store. Screenshots
|
||||
are local failure/request artifacts and are never automatically sent to an LLM
|
||||
or provider.
|
||||
|
||||
`captureEvidence` may return a partial/unavailable bundle. Console summaries are
|
||||
bounded level/message/count records. The pinned browser-harness 0.1.13 source
|
||||
supports `drain_events()` and enables Page/DOM/Runtime/Network on tab attach;
|
||||
implementers may consume buffered `Runtime.consoleAPICalled` and
|
||||
`Network.loadingFailed` events only if present. Do not invent a helper method or
|
||||
claim complete console history. Network summaries contain method, origin-only
|
||||
URL, status/category—never query/fragment tokens, headers, cookies, or bodies.
|
||||
Console text can still contain arbitrary page content; local/private and bounded
|
||||
is not the same as secret-free, so public shaping must redact it.
|
||||
|
||||
## File ownership / update order
|
||||
|
||||
- `contracts.ts` — shared API owner (this change); additions must remain typed.
|
||||
- `bridge.ts` — later JSONL transport validation and request deadlines; no new
|
||||
public operation names without updating `contracts.ts` first.
|
||||
- `browser_worker.py` — later pinned browser-harness adapter and persistent
|
||||
target/diagnostic state; use only observed harness capabilities.
|
||||
- `browser-session.ts` — browser process/profile/CDP ownership and parallel
|
||||
session isolation; it does not own tab protocol state.
|
||||
- `controller.ts` — later bounded action orchestration and automatic
|
||||
failure-evidence call; preserve stale/no-blind-retry mutation semantics.
|
||||
- `index.ts` — later public redaction, tool/CLI wiring, and origin narrowing;
|
||||
never expose private artifact contents or raw URL paths.
|
||||
- `fb-scenarios-ci` — scenario file parsing, trusted argv/config hooks, CLI,
|
||||
reports, and CI examples. Composition requires an explicitly supplied model
|
||||
adapter and returns a capability error when absent; never silently selects a
|
||||
provider.
|
||||
|
||||
Existing test mocks that implement only `BrowserBridge` need no compatibility
|
||||
aliases; runtime mocks must implement every `BrowserRuntime` method. The
|
||||
scenario worker should import the exports above rather than define alternatives.
|
||||
@@ -0,0 +1,141 @@
|
||||
# Fast-browser development-readiness backlog
|
||||
|
||||
Status: **validated native-Linux acceptance complete; final offline recheck passed** (2026-09-20). This backlog records the validated scope, not a claim of non-Linux or universal browser readiness. The original M1 visible-cleanup run failed closed and is preserved as historical evidence; the validator later completed the root-first cookie experiment, immediate-popup flow, full M1/development matrix, managed CLI scenarios, assertion path, and latest deadline gate with cleanup and empty process/port census. Linux-only supervision and non-atomic PID identity proof remain explicit constraints; fail closed when ownership is uncertain and do not claim SIGTERM guarantees universal all-app durability. The current index implements the compact eight-tool Pi catalogue and typed `fast_browser_control` payload. Use only the handed-off CLI commands and do not infer behavior beyond the current schemas.
|
||||
|
||||
## Final native-Linux validation checkpoint (2026-09-20)
|
||||
|
||||
Integrated acceptance passed for the validated native-Linux scope. Typecheck passed. Full-repository offline validation: **805 total, 803 passed, 2 skipped**. Fast-browser TypeScript: **222 total, 220 passed, 2 skipped**. Pinned Python 3.12.13: **42/42 passed**. Packaging: **4/4**, links **8/8**, package contents **36 entries**, and tracked/untracked whitespace checks clean. All live gates passed, including the latest deadline rerun: it naturally exited `1` after 3090 ms with `retry=never` and `mutationState=not_attempted`, emitted explicit private unavailable evidence, passed cleanup, and left empty process/port censuses. The deadline report is `/tmp/fast-browser-deadline-rerun-iaE8de/reports/overall-deadline-no-popup-rerun.json`; the older failed report remains preserved.
|
||||
|
||||
`imports:check` was **not run** because it requires private network access. Remote GitHub CI was **not run**; automatic CI remains configured, and equivalent managed-CLI live validation passed locally. These are scope caveats, not failures of the native-Linux acceptance gate. Non-Linux supervision remains unsupported, PID identity proof is non-atomic, and no universal SIGTERM flush/all-app durability guarantee is claimed.
|
||||
|
||||
## Baseline and guardrails
|
||||
|
||||
Observed reusable behavior includes `fast_browser_open`/`close` session ownership, exclusive persistent named-profile locks, disposable launch profiles, explicit headless launch, visible manual setup without CDP, per-session serialization, origin `"*"` as the default, autosaved global/project settings with scope boundaries, stale observation rejection, exact fill verification, click URL/visible-text postconditions, bounded worker startup/action polling, bridge request deadlines, and unknown-mutation/no-blind-retry reporting. This inventory comes from `extensions/fast-browser/browser-session.ts`/`profiles.ts`, `browser_worker.py`/`bridge.ts`, `controller.ts`, `contracts.ts`, and their offline tests; `fixtures/form.html` and `fixtures/login.html` are the current owned pages. Existing waits are concrete: browser spawn/readiness 10s, worker daemon readiness 5s, bridge requests 10s, worker action verification 3s, and controller runs 60s by default/120s maximum. The readiness work must extend those boundaries rather than replace working waits or weaken them.
|
||||
|
||||
Every live check in this backlog uses an owned local HTTP fixture and a unique run-scoped test profile (for example, a temporary `XDG_DATA_HOME` plus `fb-readiness-<run>-<worker>` names). It must use per-call/project-temporary settings, never edit global configuration, never use the active-user `harness-test` profile, and never log in to EVE or any other real service. Credential tests use only the disposable fixture account and environment/file references; values must not appear in traces, evidence, reports, or model requests. Cleanup is required on pass, failure, cancellation, and process interruption.
|
||||
|
||||
## Architecture checkpoint (scout findings)
|
||||
|
||||
The architecture scout traced `index.ts` → `browser-session.ts` → `browser_worker.py` → `controller.ts`, with configuration in `config.ts`, leases in `profiles.ts`, IPC in `bridge.ts`, and public result shaping in `index.ts`. Final contracts are published in `extensions/fast-browser/contracts.ts` with the companion `docs/fast-browser-development-contracts.md`; scoped typecheck passed. This closes the contract-publication checkpoint; M1–M5 implementation and the integrated native-Linux live acceptance gate are complete, with the final offline repository recheck now passed. The historical scout snapshot recorded `Observation`, `Postcondition`, `StepTrace`, and `RunResult`, an exactly-one-tab worker, and no public navigation/tab/popup/evidence API; the current SDK/runtime handoff has since added bounded session controls and evidence plumbing. The scenario source/CLI handoff and retention/unsupported-`compose` preflight are complete; Task 14 report/formatter/cleanup/session/artifact-redaction behavior passed the integrated native-Linux gate. These historical and current facts are not proposed APIs.
|
||||
|
||||
The main task assigned `fb-development-contracts`, which published the SDK/runtime contracts before parallel implementation. The remaining review must preserve readiness phase diagnostics (spawn, DevTools, worker, eligible-tab selection, origin approval, page/action readiness), same- versus cross-document navigation semantics, and popup/new-tab ownership. New read/assert/control/evidence features use the session-scoped runtime because the former per-operation worker lifecycle did not retain tab handles or console history. Review must preserve bounded post-close exit diagnostics and scenario/report privacy without exposing full URLs, page content, field values, authentication headers, request bodies, or provider responses. The current runtime index implements the compact eight-tool Pi catalogue and its discriminated control union; no public origin-narrowing tool exists because `settings.allowedOrigins` is per-call. Task 13 runtime cleanup/navigation/lane/evidence fixes passed the integrated native-Linux gate; the shared `BROWSER_EVIDENCE_OPERATIONS` contract is published. Task 14 CLI report/formatter/cleanup/session/artifact-redaction fixes and the managed-fixture examples passed the integrated gate; the final rerun carries real deadline metadata.
|
||||
|
||||
## Known failures to carry forward
|
||||
|
||||
- The successful historical live demo used the old attach-only, approval-based contract; it remains historical context and is not evidence for the new unattended launcher. The current native-Linux launcher is covered by the completed live gate.
|
||||
- Two managed visible Chromium launches crashed with SIGSEGV under an older flag/environment combination. A minimal user-data/CDP/no-first-run baseline worked in a diagnostic, but the integrated launcher and the cause of the old crash are not validated.
|
||||
- A browser crash was historically reported only as `Unknown browser session`; startup/worker paths had generic or empty-looking errors, including a no-approved-tab condition. The worker now has bounded sanitized startup IPC/origin diagnostics, and recent-exit diagnostics are implemented/unit-tested with safe known-closed-ID reasons; the native-Linux live gate passed.
|
||||
- The historical compatibility worker required exactly one eligible web tab and had no navigation, reload, popup, screenshot, console, or failed-request evidence surface. The current SDK/runtime handoff adds bounded session controls and evidence plumbing; the compact public wiring and final remediation passed the native-Linux gate. These historical gaps are not reasons to invent unbounded waits.
|
||||
- The historical auxiliary-model failure (unsupported Codex Spark entitlement) is an environment/provider issue, not browser reliability evidence; live readiness defaults to exact-name/literal fixture checks.
|
||||
|
||||
## Early-review acceptance status (historical checkpoints; 2026-09-20)
|
||||
|
||||
These historical code/test handoffs preceded the final native-Linux acceptance gate; their counts remain dated checkpoint evidence, not moving completion totals:
|
||||
|
||||
- `fb-transport-evidence` delivered wire/details/parent-abort and artifact-store fixes in a 22m31 handoff; 19 targeted TypeScript tests plus 35 Python tests pass.
|
||||
- `fb-runtime-tools` delivered the runtime factory/tools/controller handoff in 4m39; 48 targeted tests plus bridge coverage pass.
|
||||
- The typed wire mismatch, structured-error parser loss, dropped parent signal, and post-dispatch `not_attempted` mutation-state findings are implemented and unit-tested.
|
||||
- Bounded opaque-tab references are implemented with static validation/resolution, closed/missing checks, popup JSON coverage, 20 focused tests, clean typecheck, and passing `--check` examples. The CLI handoff now provides `node --experimental-strip-types extensions/fast-browser/scenario-cli.ts --check --format json <scenario.json>` and `bun run fast-browser:scenario -- --format human --report ./report.json <scenario.json>`. Unsupported-`compose` preflight now runs before hooks/browser startup in both `--check` and execution; Task 14 report fixes have an implementation checkpoint with 24 tests (not final). Defensive human/JSON formatters, cleanup/session handling, artifact metadata redaction, usable retained paths, explicit atomic `0600` reports in owned `0755` parents, private artifact roots, and post-cleanup retention are covered. Managed CLI implementation is complete with three `--check` examples; all three managed CLI examples and the intentional failed-assert path have REAL pass/expected-failure proofs, automatic managed-fixture CI is configured, and the final rerun includes real deadline metadata.
|
||||
- The artifact-root chmod issue is fixed with private owned roots and tests; independent review found no remaining chmod/shared-parent bypass. The separate provider-reliability TODO remains open.
|
||||
- Consolidated lifecycle status: the original M1 visible-cleanup run failed closed and preserved `/tmp/fast-browser-live-validation-5HvmBr`; the validator safely terminated all 11 verified owned processes, left no target-profile processes, and preserved `evidence/owned-process-cleanup.json` as historical evidence. PID/helper safety independently cleared; the root-first cookie experiment, full M1/development matrix, and latest deadline gate passed with complete cleanup and empty process/port census. Supervision is Linux-only; PID identity proof is non-atomic, so ownership uncertainty must fail closed. This evidence does not claim SIGTERM guarantees universal all-app durability.
|
||||
- Task 12 Python evidence privacy is complete: independent reproduction and six real-development cases pass; no buffered-event blocker remains, and exact historical/live results remain owned by the validator documentation. Timeout-evidence producer and CLI privacy are independently cleared with explicit unavailable/partial metadata, bounded independent capture, and no reopen/retry. Task 13 runtime cleanup/navigation/lane/evidence fixes and the public URL fix passed the integrated native-Linux gate; the shared `BROWSER_EVIDENCE_OPERATIONS` contract is published. The immediate-popup fix passed independent review and the REAL immediate flow without a delay workaround. Task 14 CLI report fixes passed the integrated gate. Do not treat these native-Linux results as non-Linux or remote-CI claims.
|
||||
|
||||
|
||||
## Root widget delivery sequence
|
||||
|
||||
These are the user-approved work items; all five feature milestones passed the integrated native-Linux LIVE gate. Task 1 is the published contract/documentation checkpoint; Tasks 2–6 have live acceptance evidence. The final offline repository recheck also passed; remote GitHub CI was not run and remains an explicitly unclaimed execution path.
|
||||
|
||||
1. **Docs/contracts — published checkpoint (`fb-development-contracts`):** final contracts/backlog are published and scoped typecheck passed.
|
||||
2. **Live profiles/headless/parallel validation — complete for validated native Linux (`fb-live-baseline`):** PID/helper safety, root-first cookie persistence, full M1/development coverage, latest deadline behavior, cleanup, and empty process/port census passed. Retain the historical failed-cleanup evidence; Linux-only supervision and non-atomic PID identity proof remain constraints.
|
||||
3. **Bounded page reading/assertions — complete for validated native Linux (`fb-python-development`).**
|
||||
4. **Failure evidence/diagnostics — complete for validated native Linux (`fb-transport-evidence`).**
|
||||
5. **Navigation, reload, tabs, and popups — complete for validated native Linux (`fb-runtime-tools`).**
|
||||
6. **Declarative scenarios, fixture resets, reports, and headless CI — complete for validated native Linux (`fb-scenarios-ci`).**
|
||||
7. **Independent review and final offline check — complete:** runtime/public-URL review, validator freeze, full live matrix, and final offline repository recheck passed for the validated native-Linux scope.
|
||||
8. **M1 acceptance closeout — complete for validated native Linux:** historical failures remain preserved and non-Linux/universal durability claims remain out of scope.
|
||||
13. **Final TypeScript/runtime remediation review — complete for validated native Linux:** registry, abort/error-shaping, schema-alignment, navigation, lane, evidence, and public-URL fixes passed; exact historical validation results remain owned by the validator documentation.
|
||||
14. **CLI report/formatter remediation — complete for validated native Linux:** defensive human/JSON formatting, cleanup/session/artifact metadata redaction, retained paths, report permissions/retention, managed scenarios, and expected-failure behavior passed; remote GitHub CI execution was not run.
|
||||
|
||||
## Prioritized milestones
|
||||
|
||||
### M1 — real profiles, headless, and parallel live validation (**complete for validated native Linux**)
|
||||
|
||||
**Goal:** prove the new session manager against real installed browser processes, not only fake children.
|
||||
|
||||
**Acceptance criteria**
|
||||
- Two simultaneous sessions use distinct unique named profiles, processes, CDP ports, and browser state. Closing or crashing one cannot close, unlock, or alter the other.
|
||||
- A named profile can be closed and reopened for the same local fixture without losing its test marker; profile locks reject contention and stale DevTools metadata is handled only after ownership is acquired.
|
||||
- At least one installed Chromium-family browser passes a headless login-fixture smoke test and a visible diagnostic smoke test. Vivaldi is tested when installed and never silently substituted; unavailable products produce an explicit skip/evidence record.
|
||||
- The check exercises literal/exact-name steps by default, so the readiness run needs no provider call. It records browser/version, profile/port ownership, cleanup, and any exit reason without credentials.
|
||||
|
||||
**Current live evidence:** `docs/fast-browser-live-validation.md` is validator-owned and preserves the exact historical results, including the failed original visible-cleanup run at `/tmp/fast-browser-live-validation-5HvmBr` and the report of 10 owned Chrome processes. The validator subsequently safely terminated all 11 verified owned processes, left no target-profile processes, and preserved `evidence/owned-process-cleanup.json`. The root-first cookie experiment, full M1/development matrix, managed scenarios, assertion path, and latest deadline rerun passed with cleanup and empty process/port census. The latest deadline rerun naturally exited `1` after 3090 ms with `retry=never` and `mutationState=not_attempted`, emitted explicit private unavailable evidence, and is not a live-gate failure. Its report is `/tmp/fast-browser-deadline-rerun-iaE8de/reports/overall-deadline-no-popup-rerun.json`; the older failed report remains preserved. Linux-only supervision and non-atomic PID identity proof remain constraints, and no SIGTERM all-app durability guarantee is claimed.
|
||||
|
||||
**Tests/dependencies:** retain existing `browser-session`, `profiles`, bridge, and index tests. The owned live-validation workflow and final offline repository recheck passed for native Linux; preserve the historical failed-cleanup evidence and require complete cleanup on future runs.
|
||||
|
||||
### M2 — bounded visible-page reads and assertions (**complete for validated native Linux — `fb-python-development`**)
|
||||
|
||||
**Goal:** add a read-only, caller-bounded way to inspect/assert visible fixture state without turning page text into instructions.
|
||||
|
||||
**Published contract:** the session runtime exposes typed `read` and `assert` operations with bounded visible text, URL/title/visible-text assertions, and session-scoped target identity. Implementation must preserve limits for characters, nodes, elapsed wait, origin, and result shape; reuse the existing typed bridge and 3-second postcondition/read-assert wait model rather than inventing another. Do not expose arbitrary CDP, JavaScript, DOM handles, hidden text, current input/password values, or page HTML.
|
||||
|
||||
**Acceptance criteria**
|
||||
- Reads return only bounded visible text/metadata from the approved main page and redact/truncate deterministically. Hidden, disabled, password-value, iframe, and rich-control content cannot leak through the result.
|
||||
- Assertions support at least the fixture's visible text and URL-origin outcomes, fail closed on ambiguity/staleness, and distinguish unmet from timed-out/unknown outcomes. Existing action postcondition polling and run/request deadlines are reused and documented, not duplicated.
|
||||
- Offline fake-backend tests cover visibility, limits, navigation/frame identity changes, cancellation, and no credential/page-value leakage; local live tests prove the same results on owned fixtures.
|
||||
|
||||
**Dependencies:** M1 session/fixture runner and the published runtime contract.
|
||||
|
||||
### M3 — evidence and crash/exit diagnostics (**complete for validated native Linux — `fb-transport-evidence`**)
|
||||
|
||||
**Goal:** make failures actionable without leaking page data.
|
||||
|
||||
**Acceptance criteria**
|
||||
- A bounded failure evidence bundle is captured automatically on failure and can also be requested explicitly. Local-only screenshot artifacts are the default; the bundle may also include sanitized console errors and sanitized failed-request summaries, tagged to session/step/phase. Successful runs do not capture evidence unless requested.
|
||||
- Evidence stays in bounded, local, private artifacts with explicit size, retention, and permissions. Structured console/request evidence never includes authentication headers, request bodies, credential/field values, query/fragment tokens, or unbounded page text. A screenshot is rendered fixture pixels, so it remains local/private and bounded; it is never automatically forwarded to a model or provider.
|
||||
- Startup/worker failures distinguish no eligible tab, ambiguous tabs, origin rejection, missing/mismatched harness, timeout, and browser exit. Recent-exit diagnostics now safely classify known closed IDs with a 30-second TTL and 32-entry cap, omit stderr/URLs/arbitrary errors, return unknown for expired IDs, and expose no public history API; this is implemented/unit-tested but still needs live validation.
|
||||
- Screenshot/console/request capture is failure evidence, not an autonomous vision/control surface. Evidence creation never retries an action or changes the browser state.
|
||||
|
||||
**Current M3 acceptance status:** Task 12 Python evidence privacy and the timeout-evidence producer/CLI privacy review passed. Explicit unavailable/partial metadata, bounded independent capture, and no reopen/retry were validated. No buffered-event blocker remains; exact historical/live evidence is maintained in the validator-owned documentation. Recent-exit diagnostics remain implemented/unit-tested with safe bounded output; the native-Linux live gate and final offline repository recheck passed.
|
||||
|
||||
**Tests/dependencies:** contract tests for redaction/size/permissions and crash races; induced local-fixture console/request failures; controlled browser-exit diagnostics; and M1's real session IDs passed within the validated native-Linux gate. Keep generic extension-facing errors safe while retaining the worker's already-bounded startup detail.
|
||||
|
||||
### M4 — navigation, reload, tab selection, and popups (**complete for validated native Linux — `fb-runtime-tools`**)
|
||||
|
||||
**Goal:** replace the current exactly-one-eligible-tab limitation with explicit, caller-authorized lifecycle controls.
|
||||
|
||||
**Published SDK/runtime and Pi contract:** typed `navigate`, `reload`, `listTabs`, `selectTab`, `waitPopup`, and `closeTab` methods use opaque session-scoped tab identities; popup waits never auto-select. The public `fast_browser_control` payload is a discriminated `operation` union: `navigate`/`reload` carry optional `tabId` (and `navigate` carries `url`), `listTabs` carries no request, `selectTab` carries `tabId`, `waitPopup` carries `openerTabId`, and `closeTab` carries `tabId`; all are wrapped with `sessionId` and optional `settings`. Per-call `settings.allowedOrigins` narrows policy; there is no public narrow-origin tool. Implementation/review must preserve selection lifetime, document identity, bounded completion/failure semantics, popup ownership, and explicit target choice. A model must not infer or invent a target from page text.
|
||||
|
||||
**Acceptance criteria**
|
||||
- Navigation accepts only bounded HTTP(S) destinations allowed by the session policy; reload has a bounded completion result; both invalidate observations and report redirects/out-of-origin transitions safely.
|
||||
- Tab listing/selection is deterministic and origin-filtered. Multiple eligible tabs remain an error until the caller selects one; no arbitrary tab or personal browser tab is chosen.
|
||||
- A fixture-created popup/new tab can be listed, selected, asserted, and closed only when owned by the session. Closing a session leaves externally attached browser processes untouched. The immediate-popup fix has passed independent review and a REAL immediate flow; no delay workaround is part of the contract.
|
||||
- Offline tests cover stale tab/document identities, concurrent session lanes, popup races, cancellation, and no cross-profile/tab leakage; live tests use only local fixtures.
|
||||
|
||||
**Dependencies:** M1 session ownership, M2 assertion/read result limits, and M3 diagnostics for failed navigation/popup startup.
|
||||
|
||||
### M5 — reusable scenarios, fixture/reset hooks, reports, and headless CI (**complete for validated native Linux — `fb-scenarios-ci`**)
|
||||
|
||||
**Goal:** turn the checks above into repeatable regression evidence.
|
||||
|
||||
**Current M5 status and acceptance boundary:** the user-approved scenario contract includes a managed devserver with bounded HTTP readiness in JSON scenarios; an external-server option remains supported. The managed CLI examples and intentional failed-assert scenario have REAL pass/expected-failure proofs; the final deadline rerun includes real deadline metadata, explicit private unavailable evidence, no retry, and cleanup pass. Retention, unsupported-`compose` preflight, report/formatter/cleanup/session/redaction behavior, and the integrated native-Linux acceptance gate passed. Automatic managed-fixture CI is configured, but remote GitHub CI execution was not run and is not claimed.
|
||||
|
||||
**Acceptance criteria**
|
||||
- A small local fixture suite covers form/login, navigation/reload, a popup, delayed visible state, console error, and controlled failed request. Each scenario has explicit setup, bounded steps/assertions, reset, and cleanup hooks; reset is test-only and never deletes a user-managed named profile.
|
||||
- JSON scenarios may provision an owned managed devserver and wait for bounded HTTP readiness; an explicitly configured external server remains supported. Managed CLI implementation, helper integration, automatic CI configuration, and the listed REAL proofs passed within the validated native-Linux scope.
|
||||
- Reusable tests are declarative JSON scenario definitions supplied through explicit user-provided scenario file paths and executed by a CLI runner, with schema validation before launch. The runner can execute one scenario or a parallel matrix with unique profile/port/temporary-data names in visible and headless modes. It is network-free except for the owned local fixture and never requires provider credentials.
|
||||
- Reports have human-readable and machine-readable forms with scenario/status/timings, browser/profile ownership, evidence references, failure phase, and cleanup outcome—but no credential values, field values, raw URLs with secrets, or unbounded page text.
|
||||
- CI is configured to run deterministic offline tests and headless JSON-scenario CLI workflows for relevant fast-browser PR, push, dependency, and workflow changes, with a manual trigger as well; this validation did not execute remote GitHub CI. Missing optional Vivaldi is reported as a skip, not a fallback. Firefox remains deferred.
|
||||
|
||||
**Dependencies:** M1–M4 contracts and evidence, fixture server/reset design, the available CLI commands/hooks/reports source, automatic CI workflows with isolated temporary directories that execute the real headless JSON-scenario runner against the managed fixture, and the final native-Linux acceptance evidence. All five milestones are complete for the validated scope; remote GitHub CI execution remains unclaimed.
|
||||
|
||||
## Explicit non-goals and deferred scope
|
||||
|
||||
Firefox needs a separate backend and remains deferred; reject it honestly rather than substituting Chromium. Rich-text/contenteditable controls, iframe traversal, uploads/downloads, arbitrary JavaScript/CDP, remote browsers, screenshots as vision input, broad autonomous planning, blind retries, and production/real-EVE workflows are out of scope unless a later approved milestone specifically requires them. Parallel browser processes do not isolate server-side accounts. No speed or arbitrary-site reliability claim is made by this backlog.
|
||||
|
||||
## First milestone: actionable tasks
|
||||
|
||||
1. Reload the extension in a clean process and run the existing opt-in local fixture preflight with a temporary config/data root.
|
||||
2. Add the real-browser runner's unique profile/port allocator, process/version preflight, signal-safe cleanup trap, and machine-readable pass/fail/skip record.
|
||||
3. Run two named-profile sessions concurrently (headless plus visible where available) against the local login fixture using only exact-name literal/fixture credentials.
|
||||
4. Reopen one profile, verify its local test marker, close it, and prove the other session remains usable; capture bounded exit/cleanup diagnostics for any failure.
|
||||
5. Record browser availability, commands, sanitized errors, and evidence in the readiness report; do not edit global config or use active-user/real-service profiles.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Fast Browser live demo
|
||||
|
||||
The real Pi `fast_browser_observe` and `fast_browser_run` tools completed the harmless local form demo, using Typesafe JEV for semantic control selection and `openai-codex/gpt-5.6-luna` for greeting composition. The auxiliary Codex request explicitly used `reasoningEffort: none`. Every mutation received actual Pi UI confirmation.
|
||||
|
||||
## Environment
|
||||
|
||||
- Dedicated Chrome profile, local CDP endpoint `http://127.0.0.1:9227`.
|
||||
- Local fixture `http://127.0.0.1:8767/form.html`; only that origin allowed.
|
||||
- Separate project-local Python environment with browser-harness0.1.13.
|
||||
- No hosted browser service. JEV and Pi-model inference used remote APIs.
|
||||
|
||||
## First attempt and recovery
|
||||
|
||||
JEV selected the display-name input and the approved literal `Ada` fill was verified. JEV then selected the greeting field, but Codex Spark rejected generation because that model was unsupported with the current ChatGPT-account authentication. No greeting fill or Continue click occurred in that failed attempt.
|
||||
|
||||
The user explicitly selected GPT5.6 Luna with reasoning disabled. The launcher and auxiliary-call reasoning policy were updated; the remaining two steps were resumed without repeating the name fill.
|
||||
|
||||
## Successful remaining two-step run
|
||||
|
||||
| Stage | Greeting fill | Continue click |
|
||||
| --- | ---: | ---: |
|
||||
| Observation | 148 ms | 11 ms |
|
||||
| JEV decision | 374 ms | 207 ms |
|
||||
| JEV confidence | 0.90 | 0.86 |
|
||||
| Luna text generation | 2,042 ms | — |
|
||||
| User approval | 1,594 ms | 1,692 ms |
|
||||
| Action and verification | 11 ms | 8 ms |
|
||||
|
||||
Total elapsed for these **remaining two steps**: **6,087 ms**, including **3,286 ms** of human approval time. Excluding approval wait gives approximately2,801ms, but this is not a separate autonomous benchmark. It excludes the earlier name fill, failed Spark attempt, setup and diagnostics.
|
||||
|
||||
Results:
|
||||
- Generated greeting field value verified.
|
||||
- Continue clicked once, then previously absent visible text `Fixture complete` verified.
|
||||
- Both mutations reported `mutationState: verified`; no uncertain action outcome remained.
|
||||
- No blind retries or silent model substitution.
|
||||
|
||||
The generated greeting was displayed for user approval and in the dedicated browser. This report intentionally does not duplicate field contents.
|
||||
|
||||
## Conclusions and next work
|
||||
|
||||
This validates the end-to-end integration: local observation, constrained JEV selection, separate Pi-model composition, exact-action approval, local execution and deterministic postcondition checks. It does not establish reliability on arbitrary websites, population-level confidence calibration or a speed advantage over another agent.
|
||||
|
||||
Next priorities:
|
||||
1. Replace environment-only non-secret settings with project-local/runtime configuration so model changes do not require restarting Pi.
|
||||
2. Compare JEV and a small-model-only selector on identical candidate sets and tasks.
|
||||
3. Measure repeated-run success, p50/p95 latency, startup/observation overhead and fallback cost, separating human approval waits.
|
||||
|
||||
Demo setup and owned resource IDs are in ignored `.pi/fast-browser-demo/README.md`; ask the user before closing the dedicated browser.
|
||||
@@ -0,0 +1,125 @@
|
||||
# Fast-browser live validation (milestone 1)
|
||||
|
||||
`extensions/fast-browser/live-validation.ts` is an opt-in, local-only integration harness. It starts an HTTP fixture on a random loopback port, creates a unique temporary profile root and CDP lease root, and uses only the typed `BrowserSessionManager`, `BrowserBridge`, and `runSteps` APIs. It never loads normal user profiles, reads existing manual sessions, contacts JEV/provider APIs, or uses external accounts. Literal accessible names are required; the injected decider throws if a semantic decision is attempted.
|
||||
|
||||
The fixture's `/setup` response sets a synthetic, run-unique persistent cookie (`Max-Age=3600`). Its page makes a bounded `/ready?case=setup` request; the server resolves readiness only after receiving that cookie, so this is evidence that Chromium processed the seed response—not a manual credential login. The visible setup launch is asserted to have no CDP or headless flags. Automation, when unblocked, reopens the same named profile and verifies retention with a visible postcondition after a literal fill and click. Headless launch, two concurrent isolated named profiles, same-profile contention, closing one while the other remains alive, and cleanup are covered independently.
|
||||
|
||||
## Integration command
|
||||
|
||||
From the repository root, with the existing pinned environment:
|
||||
|
||||
```bash
|
||||
node --experimental-strip-types extensions/fast-browser/live-validation.ts
|
||||
```
|
||||
|
||||
The focused test is opt-in and does not start a browser during ordinary test runs. It currently fails honestly on the visible-close blocker above until the production lifecycle fix lands:
|
||||
|
||||
```bash
|
||||
FAST_BROWSER_LIVE=1 node --test --experimental-strip-types \
|
||||
extensions/fast-browser/live-validation.test.ts
|
||||
```
|
||||
|
||||
The harness requires `/usr/bin/chromium` and `.pi/fast-browser-demo/venv/bin/python` with `browser-harness==0.1.13`. It uses finite startup, action, readiness, worker, and close deadlines. Cleanup reports owned process/root leftovers rather than terminating anything it did not own.
|
||||
|
||||
## Validation evidence (2026-09-19)
|
||||
|
||||
Runtime versions: Chromium `152.0.7977.75 Arch Linux`; Python `3.12.13`; `browser-harness 0.1.13`; Node `v22.23.2`.
|
||||
|
||||
- **Blocked:** visible manual setup seeded the cookie and `/ready` proved it. `manager.close()` returned, establishing that the manager's tracked Chromium root had exited, but owned GPU descendant PID `2870508` still carried the temporary profile `--user-data-dir`; `SingletonLock` was still present and the next `manager.open()` rejected the profile. Visible automation was skipped rather than bypassing the lock. The PID is from this disposable test profile only.
|
||||
- **Passed:** headless launch with bounded literal fill/click and verified visible postcondition.
|
||||
- **Passed:** two different headless named profiles concurrently, server-verified cookie/state isolation, same-profile contention rejection, and profile B remaining alive after profile A closed.
|
||||
- **Passed:** final owned cleanup; no test processes or temporary root remained.
|
||||
|
||||
This is a production lifecycle blocker, not a fixture workaround. The minimal proposed fix is for the session manager to own/wait for the complete Chromium process tree (including GPU/renderer descendants) and confirm the profile lock is gone before releasing a named-profile lease, within its existing bounded close deadline. No production module or package manifest was changed in this milestone.
|
||||
|
||||
## Controlled Linux rerun (2026-09-20)
|
||||
|
||||
The Linux-only PID review was cleared for this controlled run. Preflight used the real `/usr/bin/chromium` and a real positive version-check PID `3222628` (parent `3222621`, not the validator process); the pinned environment remained Python `3.12.13` with `browser-harness 0.1.13`. No fake process or PID seam, user profile, or external origin was used.
|
||||
|
||||
The original M1 command was bounded with `timeout -k 5s 180s`. Cookie seed/readiness reached the visible setup close boundary, but close failed closed with `browser process-group ownership could not be confirmed; preserving session state`; automation reuse was skipped. Headless bounded fill/click and concurrent profile isolation/contention passed. Cleanup correctly preserved the uncertain owned root `/tmp/fast-browser-live-validation-5HvmBr` rather than deleting it. Ten owned Chromium processes remained at capture: `3222696, 3222709, 3222710, 3222712, 3222738, 3222740, 3222752, 3222761, 3222770, 3222771`; the root process was `3222696` and GPU descendant `3222738`. This run exited `124` from the finite timeout and is evidence for the lifecycle blocker, not a release claim.
|
||||
|
||||
The canonical development validator then ran with a bounded `240s` timeout. All six cases passed: input-safe read/assert/navigation/reload, popup/no-auto-switch/targeted-read selection, A→B evidence isolation, narrowed peer-origin filtering with base restoration, private PNG+diagnostics capture, and honest provenance handling (`Evidence provenance unavailable: false`). Its private artifacts lived under a unique `/tmp/fast-browser-runtime-live-*` root and were removed after owned cleanup; no canonical-run process/root remained. The report artifact is this document; non-Linux/global release clearance remains pending.
|
||||
|
||||
## Isolated managed-CLI batch (2026-09-20, no rerun after this evidence)
|
||||
|
||||
After the independent broker/provenance review cleared, a single sequential native-Linux batch ran with `/usr/bin/chromium` `152.0.7977.75`, Python `3.12.13`, `browser-harness 0.1.13`, and Node `v22.23.2`. `HOME`, `PI_CODING_AGENT_DIR`, XDG config/data/cache/state/runtime roots, `TMPDIR`, cwd, scenario JSON copies, and report roots were unique and owned. The four loopback ports were unique and all managed servers were started only by `scenario-cli`; no active `harness-test` profile, provider key, JEV call, or external origin was used. Synthetic `FAST_BROWSER_TEST_PASSWORD=demo-password` was local-only and did not appear in reports, stdout, or diagnostics.
|
||||
|
||||
The M1 validator reported: manual setup reached cookie readiness and `manager.close()` returned, but its stale tombstone assertion expected `/Unknown browser session/`; the manager intentionally returned `Browser session is no longer available: session closed`. Headless launch and the two-profile case then failed closed on process-group ownership, and the finite command exited `124`. The validator assertion is now corrected; the fixture now sends `Max-Age=3600`, but no post-change browser rerun has been made.
|
||||
|
||||
The preserved setup profile from that batch was inspected without launching a browser. The fixture source used at capture time had only `Path=/; SameSite=Lax` and no `Expires` or `Max-Age`, so it created a session cookie rather than a persistent cookie. HTTP receipt is nevertheless proven: `/ready?case=setup` only resolves after the server sees the exact cookie value, and the M1 run reached that readiness. After the setup close, `Default/Cookies` was SQLite `journal_mode=delete` with zero rows, `Cookies-journal` was zero bytes, no `Cookies-wal` existed, and `SingletonLock` was absent; no pending WAL/checkpoint explains the absence. The tracked root had closed and no live process for the named profile remained. Therefore this batch does **not** prove profile-cookie reuse: the zero row is explained by the old validator fixture's session-cookie attribute, while the independent root-exit-before-first-census lifecycle blocker still prevented the automation phase.
|
||||
|
||||
The three managed examples ran from disposable JSON copies. `local-form` and `local-login` completed all browser operations, but their reports were failed because owned browser cleanup did not complete before the bounded outer timeout (`124`). The corrected-port `local-popup` navigation passed, then its click failed because the example's postcondition (`Scenario fixture ready`) already held before the click; the worker correctly refused to treat that as proof of a mutation. The intentional failed-assert scenario reached `assertion timed_out` after a credential fill, retained a real PNG and diagnostics JSON, and produced a nonzero failed report. Its report was mode `0600` in an owned mode-`0755` directory; artifact files were `0600` under mode-`0700` roots; diagnostics contained `password=[redacted]` and the synthetic secret was absent.
|
||||
|
||||
Every browser and crashpad group left by the timed-out commands was later cleaned only after verifying current PID/start-time/UID/executable/argv/ancestry/PGID/SID ownership against the unique batch roots. All managed ports were closed and no batch-owned Chromium or crashpad process remained. The M1 cleanup proof is retained at `/tmp/fast-browser-next-validation-uqoyor/m1-cleanup.json`; per-scenario cleanup proofs and the complete summary are under `/tmp/fast-browser-next-validation-uqoyor/` (`batch-summary.json`). Disposable profile and retained-evidence roots remain preserved for diagnosis. This batch is validation evidence, not release clearance.
|
||||
|
||||
## Follow-up M1 gate rerun (2026-09-20, Linux; CLI batch intentionally not started)
|
||||
|
||||
A final ownership review cleared another single native-Linux run with fresh roots, `/usr/bin/chromium`, the pinned Python environment, isolated HOME/XDG/TMPDIR/cwd/config, and no provider or active user-profile state. The original M1 ran first and was **not** a full pass: visible manual setup seeded the synthetic cookie and closed cleanly, but visible automation failed its retained-session click with `Postcondition not verified; action was not retried`. Headless bounded fill/click and the two named-profile isolation/contention cases passed. The validator reported owned cleanup passed and exited `1` solely for the automation case.
|
||||
|
||||
Because M1 did not pass, the gate correctly stopped before launching any managed CLI examples or intentional-failure scenario. A post-run `/proc` census found no Chromium or crashpad process carrying the fresh batch roots, and no subsequent CLI ports existed. The batch root is `/tmp/fast-browser-final-validation-ewubUW`; its bounded M1 output is `m1.stdout`. Chromium left only disposable `org.chromium.Chromium.*` socket directories under the isolated TMPDIR; these contain no live owned process and were preserved rather than removed blindly.
|
||||
|
||||
The stale recent-exit tombstone assertion and validator cookie are now corrected in `live-validation.ts`: closed-session lookup expects `session closed`, and `/setup` sends `Max-Age=3600`. This rerun therefore does not reproduce the earlier session-cookie explanation, but it also does not prove profile-cookie reuse: the named profile was removed after owned cleanup and no retained profile database remained for inspection. The remaining failure is separate from the validator assertion and broker census fixes: Chromium processed the seed request, yet the post-close automation session did not reach the retained postcondition. No production spec was changed and no release or cookie-reuse claim is made.
|
||||
|
||||
## Independent managed-CLI rerun after the M1 gate (2026-09-20)
|
||||
|
||||
The M1 cleanup gate passed even though retained-session automation did not, so the independent CLI proof was run separately with fresh isolated state at `/tmp/fast-browser-cli-final-OqHfA6`. The environment again used only `/usr/bin/chromium`, the pinned Python environment, unique loopback ports, isolated HOME/XDG/TMPDIR/cwd/config roots, synthetic local credentials, and no provider keys or user profiles.
|
||||
|
||||
Successful managed examples:
|
||||
|
||||
- `local-form`: exit `0`; all two operations passed; cleanup passed.
|
||||
- `local-login`: exit `0`; all three operations passed using the synthetic environment credential; cleanup passed.
|
||||
- `local-popup`: the original immediate-popup copy reached the click but timed out in `waitPopup` (exit `1`, cleanup passed). The failure is retained as a production target-inventory race finding: the click postcondition passed, but the popup was not discovered by the opener-bound wait. A disposable diagnostic copy with a 100 ms `window.open` delay then passed navigation, click, popup discovery, and read with exit `0`; that delay is diagnostic evidence only, not popup clearance. The source fixture was restored to immediate popup behavior with its status-change fix preserved.
|
||||
|
||||
The intentional failed-assert scenario was changed to `wait: "none"` so the expected assertion failure remains non-aborted and evidence capture can complete. It exited `1` promptly (~506 ms), reported `assertion not_satisfied`, retained a real PNG and diagnostics JSON, and cleaned owned browser/fixture resources. The report is `0600` in the owned `0755` reports directory; artifact files are `0600` under `0700` roots. Diagnostics contain `password=[redacted]`; the synthetic credential is absent from report and stdout. Final census found no batch-owned Chromium/crashpad processes and no open managed ports.
|
||||
|
||||
The immediate-popup failure produced no retained evidence because the runtime operation aborted before the CLI's failure capture could run; cleanup still passed and the CLI removed its disposable profile. This is reported as a production/runtime evidence-retention finding, not hidden by treating cleanup success as scenario success. The validator now preserves failed M1 roots even when process cleanup succeeds and records automation cookie/outcome evidence for any future rerun.
|
||||
|
||||
## Diagnostic retained-profile rerun (2026-09-20)
|
||||
|
||||
With process cleanup verified and no popup workaround retained, only the retained-profile prerequisite and automation case were run from a fresh isolated root `/tmp/fast-browser-retained-run-gizlJ0`. No headless, isolation, popup, or managed-CLI case was launched.
|
||||
|
||||
- Setup readiness passed, named setup closed, the profile remained, and no profile process remained.
|
||||
- Setup persistence metadata was directly inspected at `.../Default/Cookies`: 20,480-byte database, zero rows, zero journal bytes, zero WAL bytes. The final post-automation metadata was identical. There was therefore no persisted Chromium cookie row to reuse; no cookie value or token was logged.
|
||||
- Automation readiness passed, but server telemetry recorded `cookieHeaderPresent: false`, `cookieReceived: false`, and `retained: false`.
|
||||
- The exact fill of `Milestone` verified. The exact click dispatched once with `mutationState: unknown` / `actionMayHaveExecuted: true`; it was never retried. Expected text was `Retained session verified`; the fixture's actual bounded outcome was `Fixture action did not meet its bounded condition`, and the controller returned `Postcondition not verified; action was not retried`.
|
||||
- Cleanup passed with no remaining owned processes. The failed diagnostic root was deliberately preserved mode `0700`, with summary mode `0600` at `/tmp/fast-browser-retained-run-gizlJ0/tmp/fast-browser-retained-diagnostic-0Rkywk/retained-diagnostic-summary.json`.
|
||||
|
||||
This proves the retained-session failure is an actual missing-cookie condition in this isolated run, not merely an unrecorded postcondition mismatch. It does not establish cross-platform or global release clearance.
|
||||
|
||||
## Immediate-popup managed rerun (2026-09-20)
|
||||
|
||||
The restored immediate-popup fixture was run alone from fresh isolated state at `/tmp/fast-browser-popup-final-8qElTH`, using native Linux `/usr/bin/chromium`, the pinned Python environment, an explicit unique loopback port, and isolated HOME/XDG/TMPDIR/cwd/config roots. The disposable scenario verified the full sequence `click → waitPopup → selectTab → read` without exposing tab IDs to the model.
|
||||
|
||||
The managed CLI exited `0` in 530 ms. Navigate, click (`Popup opened` postcondition), `waitPopup`, explicit `selectTab`, and popup read all passed; cleanup passed. Final ownership census found no batch-owned Chromium/crashpad processes and no open managed port. No popup delay workaround was used, and no timeout-metadata clearance claim is made.
|
||||
|
||||
## Controlled persistent-cookie M1 comparison (2026-09-20)
|
||||
|
||||
The current leader-first implementation was tested once on a fresh isolated Linux profile at `/tmp/fast-browser-persist-run-9J6jRV/tmp/fast-browser-persistent-cookie-m1-NKPozp`. It used `/usr/bin/chromium`, the pinned Python environment, a disposable loopback fixture, and the explicit `Max-Age=3600; Path=/; SameSite=Lax` cookie. Setup was visible and no-CDP: its process argv had no `--headless` or remote-debugging flag.
|
||||
|
||||
- Setup readiness and close passed; the server received the synthetic cookie and the named profile survived with no remaining browser process.
|
||||
- Post-close SQLite metadata showed one persistent row for the loopback host, `hasExpires: true`, `isPersistent: true`, `expiresUtc` populated, and encrypted value storage; raw cookie values were not recorded.
|
||||
- Same-profile automation received the cookie (`cookieHeaderPresent: true`, `cookieReceived: true`, `retained: true`). Fill and retained-session click both verified; actual outcome was `Retained session verified`; no action retry occurred.
|
||||
- Final metadata still showed the one persistent row. Cleanup passed with no owned Chromium/crashpad processes; the failed-case-safe root was preserved mode `0700` and the summary is mode `0600`.
|
||||
|
||||
Conclusion: root-first fixes persistent-cookie reuse in this controlled current Linux run. This is scoped evidence only; it is not global or non-Linux release clearance.
|
||||
|
||||
## Final live integration pass (2026-09-20)
|
||||
|
||||
The current tree was validated with fresh isolated Linux roots and no user profiles, providers, or fixture timing workaround.
|
||||
|
||||
- Full original M1: exit `0`; visible setup persistence, retained-session automation, headless bounded action, and two-profile isolation/contention all passed. Cleanup passed.
|
||||
- Canonical development matrix: exit `0`; all six cases passed with `Evidence provenance unavailable: false`.
|
||||
- Managed `local-form`: exit `0`; cleanup passed.
|
||||
- Managed `local-login`: exit `0`; cleanup passed.
|
||||
- Managed immediate `local-popup`: exit `0` in 531 ms; navigate, click, `waitPopup`, explicit `selectTab`, and read passed; cleanup passed.
|
||||
- Intentional assertion failure: exit `1` in 558 ms with `assertion not_satisfied`, `retry: never`; private PNG and diagnostics were retained. Report mode was `0600` in a `0755` report directory, artifacts were `0600` under `0700` roots, and diagnostics retained `password=[redacted]` only.
|
||||
|
||||
The real overall-deadline case used a no-popup `waitPopup` with a 3,000 ms scenario budget, below the operation's bounded wait. It exited nonzero (`1`) naturally in 3,066 ms; navigation passed and `waitPopup` aborted with `mutationState: not_attempted`, `retry: never`. Evidence was explicitly represented as private unavailable evidence (`status: failed`, error `browser session is unavailable`). The exact report is `/tmp/fast-browser-final-batch-boijm3/reports/overall-deadline-no-popup.json`; its scenario cleanup is `{ status: "failed", detail: "browser session cleanup failed" }`, and the top-level cleanup is `{ status: "failed", detail: "one or more owned resources failed cleanup" }`. This is the handoff finding for the CLI owner. Validation stopped immediately as required. No force-kill or mutation retry was used. A final ownership census still found no batch-owned Chromium/crashpad processes and no open managed ports, but this does not convert the reported cleanup failure into a pass.
|
||||
|
||||
The failed batch root remains preserved at `/tmp/fast-browser-final-batch-boijm3`; the complete summary is `/tmp/fast-browser-final-batch-boijm3/final-validation-summary.json`. No production or validation-source changes were needed for this pass.
|
||||
|
||||
## Deadline cleanup-race rerun after manager fix (2026-09-20)
|
||||
|
||||
After the manager/CLI cleanup-race fix, only the real overall-deadline/no-popup case was rerun from fresh native-Linux isolated state at `/tmp/fast-browser-deadline-rerun-iaE8de`; the earlier failure report was not overwritten. The no-popup fixture used a 3,000 ms scenario budget below the bounded popup wait.
|
||||
|
||||
The new report is `/tmp/fast-browser-deadline-rerun-iaE8de/reports/overall-deadline-no-popup-rerun.json`. It exited `1` naturally in 3,090 ms: navigation passed, `waitPopup` aborted, and the report preserved `retry: never`, `mutationState: not_attempted`, and private unavailable evidence (`browser session is unavailable`). Cleanup now passed with `owned runtime, artifacts, browser session, and fixture resources cleaned`. Final census found no rerun-owned Chromium/crashpad process and no open port. The rerun root remains preserved mode `0700`, and the report is mode `0600`.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Fast Browser named profiles
|
||||
|
||||
## User decisions
|
||||
|
||||
- Support both manual setup/sign-in and automated launches from `/fast-browser`.
|
||||
- Use an exclusive profile lock. Parallel agents must choose different names; copying/cloning profiles is not part of this change.
|
||||
- Keep profile data on close. There is no automatic logout, profile reset, or deletion action.
|
||||
|
||||
## Configuration and storage
|
||||
|
||||
Add optional `browser.profile`, a 1–64 character managed name starting with a letter/digit and containing letters, digits, dots, underscores or hyphens. It is not an arbitrary filesystem path and is valid only for launch mode. Without it, the existing disposable profile behavior stays unchanged.
|
||||
|
||||
Resolve profiles below `$XDG_DATA_HOME/fast-browser/profiles/<name>`, or `$HOME/.local/share/fast-browser/profiles/<name>` when XDG_DATA_HOME is unset. This matches the manual `evebot` profile command already given to the user. Adopt existing manually prepared profiles without deleting their contents, provided they are not in use. Keep directories private and reject symlink/path traversal.
|
||||
|
||||
The `browser` object continues to replace lower-precedence browser settings as a unit. An explicit per-call browser object without `profile` therefore requests a disposable profile, not an inherited persistent one.
|
||||
|
||||
## Session lifecycle
|
||||
|
||||
**Manual setup:** launch the named browser visibly with its dedicated user-data directory, but without CDP/remote-debugging flags. This mode is for the user to sign in. It does not create an automation connection, expose a page to JEV, or claim that a particular external service will accept subsequent automation. An explicit setup-session ID identifies only the owned process lifetime; observe/run cannot automate it.
|
||||
|
||||
**Automated launch:** open the same named profile with CDP enabled and return the normal automation session ID. The manual browser must be fully closed first. Browser profile locks and process ownership are distinct from CDP endpoint leases, and both must be respected.
|
||||
|
||||
Acquire a cross-process lock before either launch. Reject active unmanaged Chromium profile use as well, rather than forwarding a new command into an existing unowned browser. Do not blindly remove stale SingletonLock files. Ignore/remove stale DevToolsActivePort metadata only after obtaining exclusive profile access, so a previous session's port cannot be mistaken for the newly launched browser.
|
||||
|
||||
Closing a session terminates its owned process and releases locks only after exit is confirmed. Disposable profiles are cleaned up; named profiles survive both normal close and startup failures. Preserve retryable ownership bookkeeping if termination or cleanup fails. Shutdown handles manual and automated sessions.
|
||||
|
||||
## Launch baseline from live testing
|
||||
|
||||
The initial visible `/usr/bin/chromium` launch with extra disable/password/keychain flags and a reduced desktop environment crashed twice with SIGSEGV. User-confirmed normal launch worked. A diagnostic launch also worked with only:
|
||||
|
||||
- `--user-data-dir=<private directory>`
|
||||
- `--remote-debugging-address=127.0.0.1` and `--remote-debugging-port=0` for automation
|
||||
- `--no-first-run` and `--no-default-browser-check`
|
||||
- the requested URL
|
||||
|
||||
That diagnostic retained `XDG_SESSION_TYPE` and other non-secret desktop context. Adopt this minimal baseline for the managed launcher, omitting the two debugging flags for manual setup. Do not force a different password-store/keychain mode between setup and automation. This establishes a working combination, not which individual old flag or omitted variable caused the crash.
|
||||
|
||||
## UI
|
||||
|
||||
Add a profile row to the native subdued-value settings view. An unset name means disposable; inherited names must remain visibly inherited. Add explicit manual-setup and automated-launch menu actions. Use current saved settings and report the returned session ID/profile. Following user feedback, every valid scope-local settings edit saves automatically; Escape returns without discarding earlier edits. Cancelling a field edit or failing validation/write preserves its previous value. Launch remains an explicit separate action. Existing close-all acts only on this Pi instance's sessions and never deletes a named profile.
|
||||
|
||||
## Verification
|
||||
|
||||
Final integration checks passed: `bun run test:fast-browser` (111 TypeScript tests and 21 Python tests), `bun run typecheck`, `bun run test:packages`, `bun run test`, and `git diff --check`.
|
||||
|
||||
Offline coverage includes name validation, config roundtrips, cross-manager contention, adoption, persistent sentinel-file retention, forced-visible setup without CDP, stale-port handling, failure cleanup, menu callbacks, conversation session metadata and unchanged disposable behavior. Real cookies were not used. No external login attempt or resumption of the cancelled Evebot test occurred during development; a live named-profile setup remains a user-driven check after reload.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Unrestricted web-origin defaults
|
||||
|
||||
## User decision
|
||||
|
||||
The user requested no default website restriction. Origin policy is now intended to be optional, not a mandatory permission dialog or fixed test-origin list.
|
||||
|
||||
- Omitted `allowedOrigins` defaults to `"*"`: any valid HTTP(S) website.
|
||||
- Explicit `allowedOrigins: "*"` selects unrestricted web origins at a particular configuration scope.
|
||||
- A nonempty array retains the existing exact-origin allowlist behavior. Projects and individual runs can opt into restrictions.
|
||||
- Empty arrays and partial hostname wildcards are not alternate spellings for unrestricted access.
|
||||
- Internal browser URLs, local file URLs, malformed URLs and credential-bearing URLs remain unsupported.
|
||||
- A session created with an explicit restricted policy cannot be widened by a later run. An unrestricted session can be narrowed for one call.
|
||||
|
||||
Unrestricted origin policy does not authorize unrelated actions: the initiating user's task, bounded steps, mutation verification and no-blind-retry behavior still apply. It also does not add tab management: multiple eligible web tabs remain an explicit ambiguity rather than permission to choose an arbitrary tab.
|
||||
|
||||
## Retired demo environment
|
||||
|
||||
The active Pi process had inherited four variables from `.pi/fast-browser-demo/start-pi.sh`: a fixed CDP endpoint, the old localhost fixture origin, a pinned Python interpreter and the Luna auxiliary model.
|
||||
|
||||
At the user's request:
|
||||
|
||||
- Removed the four exports from that ignored helper and explicitly unset inherited copies before its `exec pi`.
|
||||
- Moved the useful Python interpreter and model settings into global `fast-browser.json`, with reasoning off, preserving existing settings and credential references.
|
||||
- Did not migrate the fixture origin or fixed CDP endpoint.
|
||||
- Did not delete browser profiles, the pinned Python environment, or provider credentials.
|
||||
- Verified the helper with an offline stub Pi executable: all four variables were absent in the new process.
|
||||
|
||||
A child shell cannot unset its parent Pi process's environment. A **clean restart through the updated helper**, rather than `/reload` alone, is needed to remove values already inherited by the current process. Deliberately configured environment defaults remain supported after this cleanup.
|
||||
|
||||
## Verification
|
||||
|
||||
Passed `bun run test:fast-browser` (126 TypeScript tests and 23 Python tests), `bun run typecheck`, `bun run test:packages`, `bun run test`, and `git diff --check`.
|
||||
|
||||
Coverage includes missing/unrestricted/restricted policies across config, native settings, session lifecycle, TypeScript bridge and Python worker. Restricted sessions cannot be widened, non-web/credential-bearing/wildcard-host URLs remain unsupported, and the settings view exposes default provenance. Tests isolate agent directories and project paths instead of reading the user's real global configuration or provider keys.
|
||||
|
||||
A read-only configuration check with the four legacy variables removed confirmed the actual project resolves origins to `"*"` from defaults, Python/model from global JSON, and retains its `harness-test` profile. No browser was launched or external login attempted.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Fast-browser performance
|
||||
|
||||
## Result
|
||||
|
||||
On the provider-free local Chromium fixture, 20 alternating pairs reduced median warm workflow time from **702 to 279 ms (60%)**, and p95 from **729.05 to 315.30 ms (57%)**. Both arms passed all 20 warm attempts and all three cold attempts. Counter collection and owned-process cleanup passed.
|
||||
|
||||
This is one local fixture on native Linux, not a general website or agent benchmark. The fixture includes four navigations, 0/10/60-control observations, replacing a display name, saving and previewing it, immediate and delayed actions, and popup discovery/closure. A final assertion checks the entered name independently of the action postconditions.
|
||||
|
||||
| Measurement | Baseline p50 / p95 | Optimized p50 / p95 |
|
||||
| --- | ---: | ---: |
|
||||
| Warm complete workflow (20 per arm) | 702 / 729.05 ms | 279 / 315.30 ms |
|
||||
| Observe 0 controls | 2 / 5 ms | 2 / 3.35 ms |
|
||||
| Observe 10 controls | 16 / 23.60 ms | 7 / 11.20 ms |
|
||||
| Observe 60 controls | 78 / 93.05 ms | 28.50 / 31.15 ms |
|
||||
| Cold startup + first observation (3 per arm) | 273 / 306.30 ms | 270 / 283.50 ms |
|
||||
|
||||
Warm-worker lifetime CDP calls fell **17,791 to 7,407 (58%)**; IPC helper calls fell **19,135 to 8,751 (54%)**. These are separate counters, not additive: CDP calls also use IPC. Counts include worker initialization, the unmeasured priming observation, all 20 flows, and teardown; cold-worker counts are not included in these totals.
|
||||
|
||||
Summed transport observation time per warm flow had medians of 228.23 versus 84.74 ms. Navigation medians were 339.46 versus 61.55 ms. Action medians were 112.42 versus 109.97 ms. The main gains are observation and readiness, not faster mutation execution. The three cold samples do not establish a meaningful startup improvement.
|
||||
|
||||
## Changes
|
||||
|
||||
- Batch value-free DOM metadata in chunks of 16, while retaining Chrome's AX discovery, roles, ordering and click names. Fill labels retain the existing DOM-only label extraction. Temporary object groups are released after each chunk; partial or failed batches use the legacy safe path.
|
||||
- Keep fresh per-target AX/DOM revalidation, geometry/occlusion checks, field verification and click postconditions. No node handles, visibility, geometry, document identity, or origin authorization are cached across operations.
|
||||
- Remove redundant domain-enabling calls already made by pinned browser-harness 0.1.13, collect tab and opener metadata in one target inventory, cache the browser's select-all modifier, and reuse the existing tab read for fill URL verification.
|
||||
- Compute scoped origin contexts locally, preserving lifecycle/serialization checks and enforcement on subsequent worker operations. This avoids an otherwise redundant JSONL dispatch and does not start a worker merely to narrow a policy.
|
||||
- Replace the harness's 300 ms readiness sleep with bounded 25 ms monotonic polling of the same `document.readyState == complete` predicate and timeout.
|
||||
- After one close dispatch, verify target disappearance with bounded inventory reads under the existing action-verification budget. A close acknowledgement is not proof that Chrome has removed the target. The mutation is never retried.
|
||||
|
||||
## Reproduce
|
||||
|
||||
From the repository root, export the unchanged baseline and run the same benchmark against both workers:
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/fast-browser-performance-baseline-773c628
|
||||
git show 773c628512fdfdf5a139ae1b27833bea97d74255:extensions/fast-browser/browser_worker.py \
|
||||
> /tmp/fast-browser-performance-baseline-773c628/browser_worker.py
|
||||
node --experimental-strip-types extensions/fast-browser/performance-benchmark.ts \
|
||||
--baseline-worker /tmp/fast-browser-performance-baseline-773c628/browser_worker.py \
|
||||
--optimized-worker extensions/fast-browser/browser_worker.py \
|
||||
--python .pi/fast-browser-demo/venv/bin/python \
|
||||
--browser /usr/bin/chromium \
|
||||
--samples 20 --cold-samples 3 --counts 0,10,60 \
|
||||
> /tmp/fast-browser-performance-20-pairs-final.json
|
||||
```
|
||||
|
||||
Supply an existing Python environment containing `browser-harness==0.1.13`; the runner does not install dependencies. Use a new output filename when retaining an earlier result. Missing or invalid counters, any failed attempt, or unverified cleanup makes the command fail. Temporary browser state is retained if cleanup cannot be proven.
|
||||
|
||||
Validation environment: Chromium 152.0.7977.75, Python 3.12.13, Node v26.7.0, browser-harness 0.1.13. Measured worker commit: `2dc359c48d042a2982de8c972008b76685cbe105`.
|
||||
|
||||
- Baseline worker SHA-256: `487673653dc82a029cb5533b20566527739f8f5fd2155e5cfbd89da818132035`.
|
||||
- Optimized worker SHA-256: `d33814efb085e56f0bd6f2b8cf0c119559b37fa562a31aea0af1ed20a5550d7a`.
|
||||
- Retained final report: `/tmp/fast-browser-performance-20-pairs-final.json`.
|
||||
- Report SHA-256: `2e6a965f2a7a82c624902606e80a0fba074d16c23f87a85725eaf95f6b32a4ff`.
|
||||
|
||||
## Measurement boundaries and retained failures
|
||||
|
||||
Baseline and optimized Python workers use the same runner, fixture and browser configuration. Warm pairs alternate AB/BA; browser/worker startup is measured separately. Density observations are timed separately from navigation. Reports retain every attempt, bounded operation names, elapsed times and counters, not page text, field values, credentials, raw CDP arguments or screenshots. These measurements do not establish provider latency, semantic accuracy, Internet-site performance or cross-platform behavior. The TypeScript origin-context optimization has call-count tests but is not measured by this Python-worker comparison.
|
||||
|
||||
The first stable 20-pair run, before the close/readiness correction, is retained at `/tmp/fast-browser-performance-20-pairs.json`: baseline 19/20 and optimized 18/20 warm attempts passed. Three popup closes were unverified. Faster inventory calls exposed a pre-existing one-shot close-verification race. Observation was already faster, but coarse navigation polling produced mixed workflow medians (445 versus 551 ms). A four-pair diagnostic with per-operation timing at `/tmp/fast-browser-performance-4-pairs-diagnostic-20260921.json` isolated the navigation cost. Deterministic fake-clock tests reproduced the close race before the fix. An earlier smoke failure is retained at `/tmp/performance-smoke-final.json`.
|
||||
|
||||
These earlier results are not discarded or pooled with the corrected implementation. The final 46-attempt run above passed without retries of failed mutations.
|
||||
|
||||
## Verification and inspiration
|
||||
|
||||
Focused tests cover protocol call counts, batch fallback/cleanup and privacy, accessible naming, stale targets, origin-context lifetime, benchmark accounting, and close/readiness deadlines. Existing real Chromium development validation covers navigation, popups, origin narrowing, private diagnostics and owned cleanup. Exact full-suite and review outcomes are recorded in [the workflow ledger](plans/fast-browser-performance.md).
|
||||
|
||||
[jev-ultrafast's performance report](https://github.com/browser-use/jev-ultrafast/blob/main/docs/performance.md) motivated reducing browser round trips. Its DOM/value snapshot and autonomous operation selection are not copied: fast-browser retains its narrower data and execution contracts.
|
||||
@@ -0,0 +1,27 @@
|
||||
# Fast Browser prototype
|
||||
|
||||
Historical first-slice plan. The [unattended-session plan](fast-browser-unattended-plan.md) supersedes its approval, credential and configuration requirements.
|
||||
|
||||
Branch: `feat/fast-browser`. Package/extension directory: `extensions/fast-browser`.
|
||||
User-approved scope: bounded local-browser Pi extension, same-host Chrome, JEV decisions, literal filling and optional small-model text generation. See `browser-use-jev-feasibility.md` for research.
|
||||
|
||||
Status: first slice implemented and offline-validated. See [validation results and remaining live-test work](fast-browser-prototype-validation.md).
|
||||
|
||||
## First slice
|
||||
|
||||
- Typed `observe`, `act`, and `close` bridge operations; no arbitrary model-generated Python/JS/CDP.
|
||||
- Explicit loopback Chrome endpoint, dedicated test profile recommended. Do not auto-open or connect to a user's default browser.
|
||||
- Extract actionable candidates with stable observation IDs and tab/frame/document identity. Revalidate before action and reject stale/changed targets.
|
||||
- A supplied sequence of bounded click/fill steps, rather than an unconstrained autonomous agent. JEV picks a candidate or abstains. Literal text bypasses generation; composition uses the configured Pi auxiliary model.
|
||||
- Every live mutation requires a Pi UI confirmation of the exact action/payload. Noninteractive sessions fail closed. Offline controller tests inject approval. We will not silently trade away approval overhead for speed.
|
||||
- TypeScript controller owns step/time budgets, cancellation, strict JEV response validation, approval, postconditions and compact timings. No blind action retries.
|
||||
- Browser-harness runs through a restricted Python worker. No cloud APIs, telemetry, update checks, recording, profile sync, arbitrary network fetch or raw execution tools exposed.
|
||||
- Companion skill and README describe experimental limits and setup. No automatic global installation/config edits.
|
||||
|
||||
## Validation
|
||||
|
||||
Offline TypeScript tests with fake browser/JEV/text adapters, Python bridge tests without Chrome, package checks and typecheck. Local live smoke tests only after explicit safe endpoint/test fixture setup; live inference only with approved use of credentials. Record actual failures encountered in `docs/todos.md` when not fixed immediately.
|
||||
|
||||
## Deferred
|
||||
|
||||
General autonomous planning, screenshot/vision fallback, automatic site-specific helper writing, DOM-independent actions, remote browser hosts, multi-browser concurrency, uploads/downloads, submission/Enter shortcuts, implicit bulk approval, and any efficacy claim without benchmarks.
|
||||
@@ -0,0 +1,63 @@
|
||||
# Fast Browser prototype validation
|
||||
|
||||
This records the original approval-based prototype and demo. Subsequent user requirements and implementation are tracked in [the unattended-session plan](fast-browser-unattended-plan.md).
|
||||
|
||||
Branch: `feat/fast-browser`. Validation date: 2026-09-19.
|
||||
|
||||
## Implemented
|
||||
|
||||
- `fast_browser_observe` and `fast_browser_run` Pi tools, lazily loaded config and per-extension serialized browser lane.
|
||||
- Bounded click/fill plans (up to 10 steps), exact-name deterministic shortcut, JEV semantic candidate selection, optional registry-backed composition, and per-action UI approval.
|
||||
- Literal input bypasses text generation. JEV is lazy: exact-name runs need no Typesafe key.
|
||||
- JEV requests contain the task, action, origin and bounded candidate labels, not page URL paths, query strings, fragments, page titles or field values. Control labels still intentionally go to the model and may contain private page-derived information; this is not universal content redaction.
|
||||
- Isolated Python worker using the installed, pinned `browser-harness==0.1.13` daemon. The worker owns its child process instead of using the harness's detached/retrying/auto-launch path. It attaches to exactly one already-open approved-origin tab, never navigates to choose a tab, and shuts down its own daemon/temporary directories.
|
||||
- Only main-frame native text inputs/textareas and supported clickable controls are in scope. No arbitrary model-authored code, screenshots, frame traversal, uploads, downloads, or submit/Enter shortcut API.
|
||||
- Every mutation revalidates tab/document/candidate identity and invalidates previous observations. Clicks require a previously unsatisfied URL/text postcondition. Fills verify exact current field value and any optional postcondition. Verification does not prove all internal application state or external side effects.
|
||||
- `mutationState` and `actionMayHaveExecuted` distinguish an unverified dispatched mutation from a run that never attempted one. Cancellation cannot undo an already dispatched browser action. Unknown outcomes require inspection, not blind retry.
|
||||
|
||||
## Validation performed
|
||||
|
||||
- Final `bun run test` passed the complete repository suite, including **56 fast-browser TypeScript tests and 18 Python tests**. `bun run typecheck` and `git diff --check` also passed.
|
||||
- Offline TypeScript tests cover controller budgets, cancellation races, deterministic selection, approval, JEV request/response validation, model routing, config, IPC limits and lifecycle.
|
||||
- Python tests cover candidate filtering, stale/changed targets, replay prevention, postconditions, origin changes, helper call contracts, owned daemon startup/shutdown, strict version guard, and non-sensitive protocol errors.
|
||||
- A real subprocess smoke check using this host's existing uv interpreter confirmed that installed browser-harness **0.1.8 is rejected before importing browser helpers or connecting to Chrome**. No upgrade was performed on that installation.
|
||||
- Workspace typecheck passes in the local dependency environment. The lockfile changes only add the new workspace; no dependency-version churn.
|
||||
- Package checks verify that runtime Python/TypeScript files, skill and fixture are included and tests/bytecode are excluded.
|
||||
- The earlier full repository run passed existing suites but caught a new confirmation-test expectation during a concurrent approval-format edit. That test was corrected to expect safely quoted display values; the final full rerun passed.
|
||||
|
||||
## Review findings fixed
|
||||
|
||||
- Removed an implicit import from a temporary source checkout.
|
||||
- Stopped forwarding the Pi environment and provider API keys into Python/daemon children.
|
||||
- Replaced detached daemon startup with explicit process ownership; handled cancellation during startup and waited for worker exit before releasing the lane.
|
||||
- Fixed candidate-name/role consistency, main-frame checks, click hit testing, and focused bulk input rather than per-character typing.
|
||||
- Rejected preexisting click postconditions, out-of-origin post-action states, and replay through other old snapshots.
|
||||
- Tightened probability normalization and rejected a selected JEV option that contradicts the reported winner. Confidence remains a separate provider-reported value with an uncalibrated prototype threshold, not a security boundary.
|
||||
- Made streamed model responses abortable and sanitized their failures; bounded direct JEV input calls as well as controller inputs.
|
||||
- Limited action evidence to fixed non-sensitive messages and exposed uncertain action outcomes in machine-readable form.
|
||||
|
||||
The review's suggestion that the controller must itself understand DOM/frame identity was not adopted: that identity stays private in the real bridge, whose `act` contract and tests enforce freshness immediately before mutation. Local approval retains the exact escaped URL/payload for informed permission; those confirmation values are not JEV request context.
|
||||
|
||||
## Not validated / next step
|
||||
|
||||
The original prototype validation did not include live mutation/inference. The subsequent demo partially exercised those paths, as recorded below. No comparative task-success or latency benchmark has been run. Initial validation found no Chrome on PATH and an older default harness installation.
|
||||
|
||||
### Follow-up: live demo preparation
|
||||
|
||||
At the user's request, Chrome was subsequently found at `/opt/google/chrome/chrome` (version150). A separate ignored project-local environment was prepared with browser-harness0.1.13 and a dedicated Chrome profile. The local fixture now includes a `Demo greeting` textarea for the requested small-model composition demo.
|
||||
|
||||
A **real read-only bridge smoke test passed**: it returned exactly `Display name`, `Demo greeting`, and `Continue`, excluding the password and disabled controls. After bridge shutdown, a CDP tab-list check confirmed that only the original fixture tab remained; the harness-owned blank tab was cleaned up. No API credentials were used and no fields were filled or clicked.
|
||||
|
||||
After restarting through `.pi/fast-browser-demo/start-pi.sh --resume`, the actual Pi tools ran. JEV selected `Display name` (confidence0.99,433ms); after UI approval, filling `Ada` was verified in14ms. It next selected `Demo greeting` (confidence0.89,324ms), but Spark generation failed before any greeting-field mutation. The full partial run took13,240ms, including11,552ms of human approval wait. Continue was not clicked.
|
||||
|
||||
A separate tool-free Pi diagnostic confirmed the provider error: `gpt-5.3-codex-spark` is not supported with this ChatGPT-account authentication. The user selected `openai-codex/gpt-5.6-luna` with reasoning off instead. The launcher was updated, and Codex auxiliary calls now explicitly send `reasoningEffort: none`; only the remaining two steps should be resumed after restart. Safe generation errors are now retained in tool results. Latest tests:58 TypeScript+18 Python and workspace typecheck pass.
|
||||
|
||||
During this follow-up, one no-key adapter test accidentally inherited the live Typesafe environment and issued a synthetic fixture request. The test helper now clears all fast-browser settings/key before each test, and all adapter tests prohibit HTTP. No actual browser page data or field values were sent by that test. This finding and the remaining environment-only reconfiguration limitation are recorded in `docs/todos.md`. Preparation/resume state and cleanup resource IDs remain in ignored `.pi/fast-browser-demo/README.md`.
|
||||
|
||||
No speedup or general-browser reliability claim is established by these tests. After the approved live demo, compare identical tasks against a small-model-only baseline.
|
||||
|
||||
The origin allowlist controls where this adapter operates; it is not a browser network firewall. A redirect or action may already have caused a network request before the adapter detects it. The JS/DOM checks are not atomic against a malicious page changing between check and input. Use a trusted, dedicated test profile for this prototype.
|
||||
|
||||
## Tooling failure observed
|
||||
|
||||
The browser-bridge subagent failed twice with a provider WebSocket error after writing partial files. The parent resumed once, then completed and tested the implementation directly. Reproduction evidence and a follow-up remain in `docs/todos.md`.
|
||||
@@ -0,0 +1,32 @@
|
||||
# Fast Browser: unattended, caller-configured test sessions
|
||||
|
||||
This supersedes the original prototype's mandatory per-action approval and password-field exclusion. The user explicitly requested an automation tool controlled by an initiating model such as Astra, including local test-account login, configuration files/TUI, parallel isolated browsers and headless execution.
|
||||
|
||||
## Contract
|
||||
|
||||
- The initiating model supplies the task, bounded steps, allowed origins and optional settings/credential bindings. Execution has no interactive approval layer, including in print mode.
|
||||
- Preserve operational correctness: step/time budgets, current target/document checks, exact fill verification, explicit click postconditions, cancellation and unknown-outcome reporting. These are not user permission prompts.
|
||||
- Credentials may be caller-provided literals, environment references, plain local files, or named top-level JSON file entries. Resolve only for the selected fill. Never insert resolved values into JEV state, generation prompts or result traces. Literal arguments remain present in the initiating Pi session's ordinary tool-call history; references avoid adding the value there.
|
||||
- Password inputs are supported like other native inputs. No blanket login/test-account ban. Hidden controls and unsupported file upload/rich-text mechanics remain capability limits, not authentication policy.
|
||||
|
||||
## Configuration
|
||||
|
||||
Precedence: per-call settings > project `.pi/fast-browser.json` > global agent-directory `fast-browser.json` > legacy environment defaults. File settings are reread for the next invocation. `/fast-browser` uses Pi's native selection/input UI to inspect/edit settings and choose project/global save scope; changes do not require restarting Pi.
|
||||
|
||||
Keep actual provider API keys out of settings. A Typesafe key may be selected through a local file/environment reference; ordinary Pi model authentication continues to use the model registry. Runtime endpoint/executable/model fields and declared credential references are caller-controlled, never selected by website content.
|
||||
|
||||
## Browser sessions
|
||||
|
||||
- Add explicit open/observe/run/close session lifecycle. The initiating agent can retain an opaque session ID across calls, so login and navigation state survive until cleanup.
|
||||
- Each launched session gets a distinct browser process, temporary user-data directory, CDP endpoint and browser-harness controller. Tabs in a shared profile are not treated as isolation.
|
||||
- Support Vivaldi, Chrome and Chromium using the current CDP/browser-harness backend. A configured product never silently falls back to another product.
|
||||
- A caller may override the executable path. Headless launch is explicit (`browser.headless: true`); visible launch remains available for debugging. An attached browser's display mode was determined externally when it started.
|
||||
- Attach to an explicit compatible endpoint when requested, without closing the externally owned browser. Hold an exclusive endpoint lease for the session to reject competing automation on that same endpoint; attachment does not isolate cookies or local storage from other clients.
|
||||
- Independent sessions can operate concurrently. Serialize operations within each session, not globally. Close only the requested owned session; extension shutdown cleans up owned resources.
|
||||
- Firefox is explicitly deferred by the user. Reject unsupported engine selection honestly; a future Firefox backend must not pretend CDP compatibility.
|
||||
|
||||
Separate browser processes do not isolate application database records, accounts or server-side sessions. Parallel suites must choose suitable test accounts/data as well.
|
||||
|
||||
## Validation boundaries
|
||||
|
||||
Use offline controller, credential, native-UI, process-lifecycle and worker tests first. Do not use modified repository code to bypass the currently loaded tool's mandatory action UI; live unattended checks require a future session with the updated tool contract. No real credentials, browsers or provider calls are required for this implementation's regression suite.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Unattended Fast Browser validation
|
||||
|
||||
Implementation follows [the unattended-session plan](fast-browser-unattended-plan.md). The original [live demo](fast-browser-live-demo.md) remains historical evidence for the prior attach-only, UI-approved implementation, not for the new launcher.
|
||||
|
||||
## Implemented and covered offline
|
||||
|
||||
- No controller approval callback or adapter UI authorization gate; literal/credential flows also work in noninteractive modes.
|
||||
- Literal, environment, plain-file and top-level JSON credential references, bounded file reads, cancellation, sanitized errors and password input support. Resolved values are not inserted into JEV input, text-generation prompts or result traces.
|
||||
- Global defaults, project overrides and per-call settings; provenance; strict validation; atomic mode-0600 writes; native `/fast-browser` editing of scope-local partial values. Clearing an override means inheritance, not secretly disabling lower-precedence configuration.
|
||||
- Optional auxiliary-model lookup is lazy. An unavailable configured model does not break literal fills. Codex reasoning-off is explicit; unsupported reasoning mappings fail rather than silently select another provider/model.
|
||||
- Explicit open/observe/run/close lifecycle, opaque session IDs, per-session busy guards and concurrent independent sessions. Caller-supplied settings cannot retarget an existing session or broaden its origin policy.
|
||||
- Vivaldi/Chrome/Chromium executable selection, explicit headless flags, unique process/profile/endpoint per launched session and credential-free child environment. Attach sessions lease the configured endpoint without owning the external browser process.
|
||||
- Independent close, cancellation, post-ready browser exit, startup failure, graceful/forced termination, pending-open shutdown races and retryable failed cleanup.
|
||||
|
||||
## Issues found and fixed during development
|
||||
|
||||
- Scope saves originally serialized merged effective values, copying project-only settings into global defaults. The editor now edits and saves only the selected scope's partial object; tests preserve the global/project boundary.
|
||||
- Closing an attached session initially rejected it because the adapter confused browser-process ownership with lease ownership. Closing now releases the session lease without killing the external browser.
|
||||
- Forced process termination originally allowed profile/lease cleanup before confirmed exit. Cleanup now retains ownership when termination cannot be confirmed and supports retry.
|
||||
- Failed startup cleanup could lose ownership bookkeeping, and pending-entry failures could disappear before shutdown inspected them. Retained failed-start resources and pending snapshots are regression-tested.
|
||||
- Earlier prototype tests once inherited a real Typesafe key and issued one synthetic fixture request, without field values or page contents. Adapter tests were changed to isolate configuration and prohibit HTTP. The current suite requires no live browser, provider or network requests.
|
||||
- The auxiliary model catalogue listed Codex Spark, but the actual provider rejected it for ChatGPT-account authentication. Model catalogue availability is not API entitlement; safe errors are retained and no automatic model fallback is used.
|
||||
|
||||
## Final checks
|
||||
|
||||
Passed after integration and review fixes:
|
||||
|
||||
- `bun run test:fast-browser`: **93 TypeScript tests and 19 Python tests**.
|
||||
- `bun run typecheck`.
|
||||
- `bun run test:packages`.
|
||||
- `bun run test` (full repository suite).
|
||||
- `git diff --check`.
|
||||
|
||||
## Settings persistence follow-up
|
||||
|
||||
The original native editor kept an explicit-save draft. User feedback clarified that leaving the editor should retain changes, as Pi's own settings do. The corrected workflow saves each valid scope-local edit immediately, makes Escape a back/exit action, and preserves previous saved values when a field is cancelled, invalid, or cannot be written. It must not copy project overrides into global defaults. This supersedes earlier explicit-save/cancel instructions.
|
||||
|
||||
## Remaining live validation
|
||||
|
||||
After `/reload`, exercise the new tool contract against the local login fixture using two headless browser sessions, including Vivaldi where installed. Verify independent login state and cleanup with actual browser builds. Offline fake-process tests do not prove a particular installed browser accepts every flag.
|
||||
|
||||
This work did not change the user's global settings, launch a new live browser, or invoke the old loaded approval-based browser tools to bypass their contract. Firefox remains explicitly deferred. Follow-up items are tracked in [todos.md](todos.md).
|
||||
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"incident": "multi_tool_use.parallel subagent completion did not wake idle orchestrator",
|
||||
"observedAt": "2026-09-03T15:54:57-04:00",
|
||||
"environment": {
|
||||
"piVersion": "0.84.4",
|
||||
"repositorySubagentPackageVersion": "0.1.0",
|
||||
"note": "The failing tool surface was the harness-provided functions.subagent tool wrapped by multi_tool_use.parallel. It has not been proven to be implemented by the repository @jbcode/pi-subagents package.",
|
||||
"parentSessionId": "01a06885-7f00-7cff-bc2f-334984151941",
|
||||
"parentSessionFile": "/home/jb/.pi/agent/sessions/--srv-data-code-jb-pi-harness-tools--/2026-09-03T18-26-14-400Z_01a06885-7f00-7cff-bc2f-334984151941.jsonl",
|
||||
"provider": "openai-codex",
|
||||
"model": "gpt-5.6-sol",
|
||||
"reasoning": "high"
|
||||
},
|
||||
"failingCall": {
|
||||
"tool": "multi_tool_use.parallel",
|
||||
"payload": {
|
||||
"tool_uses": [
|
||||
{
|
||||
"recipient_name": "functions.subagent",
|
||||
"parameters": {
|
||||
"agent": "codebase-analyzer",
|
||||
"name": "workflow-architecture",
|
||||
"cwd": "/srv/data/code/jb-pi/harness-tools",
|
||||
"task": "Read-only research. Analyze the current extensions/workflow and extensions/_workflows architecture end-to-end. Focus on what behavior is rigidly enforced, where failures/error-proneness arise, state/recovery mechanics, human gates, review gates, and extension complexity. Read relevant tests and docs. Return concrete findings with file paths and a proposed split of responsibilities between skills, minimal extensions, and ordinary repo instructions. Do not edit files."
|
||||
}
|
||||
},
|
||||
{
|
||||
"recipient_name": "functions.subagent",
|
||||
"parameters": {
|
||||
"agent": "precedent-locator",
|
||||
"name": "workflow-history",
|
||||
"cwd": "/srv/data/code/jb-pi/harness-tools",
|
||||
"task": "Read-only research. Investigate git history and docs/ADR/workflow artifacts related to the workflow extension. Identify original goals, decisions, known usability issues, failure modes, later fixes, and lessons that should shape a replacement workflow. Return commits/docs/files and concise lessons. Do not edit files."
|
||||
}
|
||||
},
|
||||
{
|
||||
"recipient_name": "functions.subagent",
|
||||
"parameters": {
|
||||
"agent": "codebase-pattern-finder",
|
||||
"name": "workflow-patterns",
|
||||
"cwd": "/srv/data/code/jb-pi/harness-tools",
|
||||
"task": "Read-only research. Find existing skills, extensions, tools, and agent instructions in this repo that already support planning, task tracking, subagent delegation, independent review, human gates, verification, worktrees, continuation, and recovery. Identify reusable pieces and missing capabilities. Return concrete paths and recommended composable patterns. Do not edit files."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"launchResults": [
|
||||
{
|
||||
"name": "workflow-architecture",
|
||||
"result": "launched",
|
||||
"childId": null,
|
||||
"childSessionId": null
|
||||
},
|
||||
{
|
||||
"name": "workflow-history",
|
||||
"result": "launched",
|
||||
"childId": null,
|
||||
"childSessionId": null
|
||||
},
|
||||
{
|
||||
"name": "workflow-patterns",
|
||||
"result": "launched",
|
||||
"childId": null,
|
||||
"childSessionId": null
|
||||
}
|
||||
],
|
||||
"expectedLifecycle": [
|
||||
"wrapper returns one prompt acknowledgement per child without waiting",
|
||||
"each child reaches one terminal state",
|
||||
"each terminal result is queued exactly once for the parent",
|
||||
"an idle parent is woken for each queued result",
|
||||
"UI and tool-visible child state agree",
|
||||
"late subagent_message resumes a completed child rather than steering a running child"
|
||||
],
|
||||
"observedLifecycle": [
|
||||
"all three wrapped launches returned successful acknowledgements",
|
||||
"the parent continued independent work and then became idle",
|
||||
"the UI later appeared to show all three children completed",
|
||||
"no terminal result woke the idle parent",
|
||||
"the user manually woke the parent and reported completion",
|
||||
"subagent_message for each child responded that the message was delivered to a running subagent"
|
||||
],
|
||||
"controlCase": {
|
||||
"tool": "functions.subagent",
|
||||
"wrapped": false,
|
||||
"name": "skill-workflow-plan-review",
|
||||
"cwd": "/srv/data/code/jb-pi/harness-tools/.worktrees/skill-led-development-workflow-plan",
|
||||
"result": "automatic completion delivered",
|
||||
"elapsed": "14m 9s"
|
||||
},
|
||||
"missingEvidence": [
|
||||
"exact launch acknowledgement timestamps",
|
||||
"internal child task IDs",
|
||||
"internal child session IDs",
|
||||
"terminal-state timestamps",
|
||||
"callback queue entries"
|
||||
],
|
||||
"reproductionRequirement": "A focused fixture must capture the missing internal evidence from instrumented execution rather than infer it from UI state."
|
||||
}
|
||||
@@ -94,7 +94,7 @@ Below you will find my preferences so that we can be more aligned as we work tog
|
||||
- Plan before we implement. Investigate the codebase, do research, and grill me with questions. I'd rather spend some time answering questions first and get a result I am happy with rather than something we'd have to do again.
|
||||
- Prototype or create PoCs for large features first. If we can identify if something will or will not work early, that will save time in the long run.
|
||||
- Do not touch production or live databases unless explicitly instructed to do so.
|
||||
- When presenting any single local file for my review, or a structured long-form report, plan, ADR, review, or summary, load the [`artifact-presenter`](../skills/document/artifact-presenter/SKILL.md) skill and open the complete file with mcat in its reusable Herdr pane instead of returning the full artifact inline.
|
||||
- When presenting any single local file for my review, or a structured long-form report, plan, ADR, review, or summary, load the [`artifact-presenter`](../skills/document/artifact-presenter/SKILL.md) skill and open the complete file through its environment-aware presenter instead of returning the full artifact inline.
|
||||
- After reporting status, if there is no input required from me, continue work on the task.
|
||||
|
||||
## Subagents and Parallel work
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
# Local alternatives to JEV: SemIf, decider, and djev
|
||||
|
||||
Reviewed: 2026-09-21. Research only; no deployment or integration decision approved.
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Trial Mapika decider first on the existing RTX 4090, then f302. Keep SemIf as a frozen general-model scoring baseline; defer djev. Do not rent a GPU just to reduce JEV costs.**
|
||||
|
||||
Only **decider** releases purpose-trained decision weights. SemIf and djev implement decision interfaces around existing open-weight models. All three can avoid autoregressive answer generation, but that alone does not establish JEV-equivalent accuracy, calibrated confidence, or faster end-to-end decisions.
|
||||
|
||||
Your hardware is sufficient to investigate the most promising small models without renting anything. Privacy and offline availability are the strongest reasons to proceed. Cost savings and latency improvements remain workload-dependent.
|
||||
|
||||
## Comparison
|
||||
|
||||
| Candidate | What it actually is | Fit for fast-browser | Main limitation | Priority |
|
||||
|---|---|---|---|---|
|
||||
| **decider** | Fine-tuned Qwen3.5 decision models plus Python inference and a JEV-shaped server | Choice supports 2–255 options; includes routing and predicate APIs; some browser-selection evidence | No validated 61-way fast-browser performance; API and confidence differences need handling | **First trial** |
|
||||
| **SemIf** | MIT scoring framework using existing frozen models, strongest tested baseline Qwen3.5-4B | Useful direct-logit baseline for small candidate sets | Python supports only 2–16 options; browser demo 2–20; scores explicitly uncalibrated | Comparison baseline |
|
||||
| **djev** | Apache-2.0 typed-decision framework over DiffusionGemma-26B-A4B-it | Nominally supports 61-option Choice, with exact-token validation | Published BF16 runtime needs roughly 52 GB just for weights; no held-out browser-quality evidence | Defer |
|
||||
|
||||
Fast-browser allows **60 controls plus `none_of_the_above`**, not just a binary decision. Its controller already accepts a small `Decision` interface: candidate ID or null, confidence, and model identity. Keep that boundary and existing execution safeguards. [1–4]
|
||||
|
||||
### decider: best practical starting point
|
||||
|
||||
Start with **decider-2b v10 BF16 on the 4090**. It has approximately 1.9B parameters and 3.5 GB of weights. On f302, evaluate **decider-0.8b BF16** first for a low-memory baseline, then 2B with bounded short inputs if memory permits. Neither GPU has project-published measurements. [2]
|
||||
|
||||
The author reports 2B v10 MiniWoB++ success of **90.9% greedy**, or 93.2% sampled, across 22 small synthetic click tasks and eight seeds. Greedy is the more relevant comparison because fast-browser selects the highest-probability option. The separate 82.7% Mind2Web result is offline candidate selection, not completed browser workflows. Neither proves performance on real 60-control pages, filling forms, missing targets, or malicious labels. [2]
|
||||
|
||||
For routing, it has full-label classification evaluations and runtime-defined descriptions. For semantic checks, Noul returns P(yes); use a three-way Choice when “unknown” needs an explicit outcome. Its temperature scaling improves calibration on its own evaluation distribution, not necessarily ours. The release also documents weak long-array index lookup, relevant because our current criteria refer to `controls[index]`. Compare the existing request with a separately evaluated variant that places role/name descriptions directly in each criterion. [1,2]
|
||||
|
||||
**Integration is not a URL substitution:**
|
||||
|
||||
- `systemone.format_answer` rounds probabilities to four decimals. A dependency-free probe with three equal probabilities returned a sum of **0.9999**, which fails our parser's `1e-6` normalization tolerance. Use backend-specific, bounded numeric validation or preserve greater upstream precision; do not weaken all validation indiscriminately.
|
||||
- The server defaults to a local training-model path and enables compile/FP8. Explicitly select released weights and begin with **BF16/eager, compile and FP8 disabled** on Ada hardware. Published optimized timings use GH200/B300.
|
||||
- State truncation is possible. Reject oversized evidence before inference rather than silently losing candidate information.
|
||||
- Its example server binds all interfaces without a documented authentication layer. Start loopback-only; remote serving requires authenticated, restricted access. [2]
|
||||
|
||||
The 35B model is not a sensible initial target: BF16 requires an 80 GB-class GPU. Its 19.6 GB NVFP4 checkpoint targets Blackwell runtimes; fitting the weight-file size inside 24 GB does not establish RTX 4090 compatibility. [2]
|
||||
|
||||
### SemIf: useful control experiment, not a replacement yet
|
||||
|
||||
SemIf reads option-token logits from frozen language models. Qwen3.5-4B is its strongest tested baseline. On a selected 102-row TypeSafe public subset, it reports **84.5% reference agreement versus 88.3% for published JEV values**. This was not a live head-to-head, is not browser accuracy, and does not establish confidence parity. [3]
|
||||
|
||||
Its native Qwen3.5 run reports approximately **8.89 GB peak CUDA allocation**, rising to 11.61 GB with shared-state parallelism. That is suitable for the 4090, not safely for f302's 8 GB GPU. Quantized browser artifacts exist, but their quality was not measured by the BF16 quality table. [3]
|
||||
|
||||
The advertised roughly 20 decisions/second comes from sharing state across many criteria. It is **not a 50 ms measurement of one fresh browser decision**. The WebGPU Qwen3.5 smoke test took about 3.27 seconds for a direct decision on an RTX 3090. Prefer a native baseline, not an assumption that the browser demo is the fastest deployment. [3]
|
||||
|
||||
Testing the full fast-browser candidate range would require extending and validating its label/readout scheme and prompt budget. Do not silently shortlist away controls just to satisfy its limit.
|
||||
|
||||
### djev: interesting architecture, wrong first deployment
|
||||
|
||||
Djev uses one structured read of DiffusionGemma rather than ordinary multi-step diffusion generation. It releases no new trained weights. Its published Linux/CUDA/vLLM composite uses unquantized BF16 and a B200 reference host. **Neither your 24 GB 4090 nor 8 GB 4060 can run that published configuration.** The desktop's system RAM does not solve the documented single-GPU VRAM requirement. [4]
|
||||
|
||||
Quantized DiffusionGemma paths exist upstream, but compatibility with djev's pinned patches, exact-token reads, and consumer GPUs is a separate porting experiment. Do not treat it as an available supported local setup.
|
||||
|
||||
Its own original BF16 development run reported model-call p50 **59 ms**, but full HTTP p50 **376 ms**, p95 **1,099 ms**, across only 12 requests. A faster historical quantized run is not the published BF16 release. These are development fixtures, not held-out quality or matched latency evidence. [4]
|
||||
|
||||
**Its Choice confidence is normalized entropy concentration, not the winner's probability.** Reusing fast-browser's 0.85 threshold would have a different meaning. Calibration is explicitly unvalidated. [4]
|
||||
|
||||
## Hardware assessment
|
||||
|
||||
Read-only system queries observed:
|
||||
|
||||
| Target | Hardware | Sensible investigation |
|
||||
|---|---|---|
|
||||
| **This host** | RTX 4090, 24 GB VRAM; 186 GiB RAM; Ryzen 9950X3D identification with eight exposed CPUs | decider-2b BF16; SemIf Qwen3.5-4B BF16 comparison |
|
||||
| **f302** | RTX 4060 Laptop, 8 GB VRAM; 14 GiB RAM, about 8.3 GiB available; Core Ultra 7 155H | decider-0.8b BF16; test 2B memory at realistic prompt sizes |
|
||||
| **CPU-only** | Existing CPU/RAM | Decider's plain Transformers path is a possible compatibility experiment; SemIf's inspected native path and djev's published path require GPUs. Competitive CPU latency is unestablished. |
|
||||
| **Rented 24 GB** | Similar capacity to the already-owned desktop | Only useful for separate always-on availability or contention, not additional model capability |
|
||||
| **Rented 80 GB+** | Potential BF16 capacity for larger models | Later quality experiment if small models fail; capacity alone does not validate djev's runtime on a different GPU |
|
||||
|
||||
Hardware fit above is an inference from requirements, **not a successful load test**. The desktop had about 3.2 GB VRAM occupied at inspection. Include existing workloads, context/workspace allocation, power state, and laptop thermal limits in measurements.
|
||||
|
||||
“Local” inference on a rented host still sends inputs off your premises and depends on its network/operator. Serving from the desktop to f302 keeps inference on your hardware but is not offline when f302 cannot reach it. Fully offline operation on f302 requires its own model and cached runtime artifacts.
|
||||
|
||||
## Economics
|
||||
|
||||
Current published JEV price is **$0.042 per million input tokens**, with free output. At 2,000 input tokens per request: [5]
|
||||
|
||||
| Requests | JEV input cost |
|
||||
|---|---:|
|
||||
| 1,000 | $0.084 |
|
||||
| 100,000 | $8.40 |
|
||||
| 1,000,000 | $84 |
|
||||
|
||||
These are **semantic requests actually sent to JEV**, not all browser actions. A unique exact accessible-name match already bypasses JEV. The current adapter sends one target question per request. TypeSafe supports sharing state across independent questions, but that is a provider capability, not batching currently implemented here. Any planner/text or recovery calls have separate costs. [1,5]
|
||||
|
||||
Rental snapshots checked on 2026-09-21, excluding storage, bandwidth, taxes, and other extras:
|
||||
|
||||
| Example | Hourly | Always-on, 730 h/month | Break-even at 2k tokens/request |
|
||||
|---|---:|---:|---:|
|
||||
| Vast RTX 4090 marketplace listing | $0.14 | $102 | 1,667 requests/hour |
|
||||
| RunPod RTX A5000 24 GB | $0.27 | $197 | 3,214/hour |
|
||||
| RunPod RTX 4090 | $0.74 | $540 | 8,810/hour |
|
||||
|
||||
Marketplace prices and capacity are volatile; these are not availability guarantees. [6]
|
||||
|
||||
`JEV cost/request = 0.042 × input_tokens / 1,000,000`
|
||||
|
||||
`Rental break-even requests/hour = hourly_price / JEV_cost_per_request`
|
||||
|
||||
For owned hardware, count **incremental measured wall power** and operational overhead. Do not charge the entire PC's existing idle consumption to one intermittent model, or mistake GPU TDP for measured inference consumption. Extra model fallbacks can outweigh savings on cheap JEV calls.
|
||||
|
||||
Scale-to-zero hosting avoids some idle rent but introduces model-loading/cold-start delays. Keeping a worker warm trades that latency back for idle cost. For this use case, benchmark owned hardware before considering rental. [6]
|
||||
|
||||
## What would establish efficacy
|
||||
|
||||
Proposed evaluation, not yet approved or executed:
|
||||
|
||||
1. **Reproducible load gate.** Pin source, weights, tokenizer, runtime, precision, and settings. Verify BF16/eager loading, memory, 61 labels, explicit abstention, full probability output, cancellation, and token-budget rejection. Test decider-2b on the 4090, then 0.8b/2b on f302. Start with one persistent worker, not a cold model per decision.
|
||||
2. **Recorded-fixture comparison, without browser actions.** Replay the same labeled observations through JEV and local adapters. JEV calls remain remote: use synthetic/redacted fixtures, or explicitly authorize any real data sent to TypeSafe. Task text and control labels can contain sensitive information even when credentials are excluded. Preserve representative AX/control shapes. Separate calibration and held-out sets by page/task family. Cover candidate counts through 60 plus abstain, absent/duplicate targets, paraphrases, distractors, option-order changes, long labels, untrusted instructions, routing labels, and yes/no/unknown predicates. SemIf comparisons must explicitly use its supported cardinality unless extended.
|
||||
3. **Measure useful accuracy.** Report correct-target inclusion, top-choice accuracy, wrong accepted choices, abstention precision/recall, coverage, calibration, and fallback cost. Calibrate thresholds per backend and workload; neither aggregate ECE nor a high confidence is an authorization guarantee. Candidate freshness and permissions remain code checks.
|
||||
4. **Measure equivalent latency.** Record cold load separately from warm request p50/p95/p99, at concurrency one first. Include serialization, local HTTP, queueing, GPU work, memory, and incremental power. Existing JEV demo calls were 207 and 374 ms; rerun a matched baseline rather than generalizing from those two historical samples. [7]
|
||||
5. **Browser gate.** Only after offline results, run bounded browser fixtures with deterministic postconditions. Preserve candidate identity, origin policy, verification, and no-blind-retry behavior. Report false actions and uncertainty, not just successful clicks. Agree acceptance thresholds before selecting a winner; no observed failures in a small sample does not prove safety.
|
||||
|
||||
Suggested first experiment: **decider-2b on the 4090 versus remote JEV using synthetic/redacted recorded fixtures**, with no production browser execution. A fully offline local-only run can use those same labeled fixtures without contacting JEV. If quality holds and warm latency improves, test f302 and consider a reusable local decision service for routing and predicates. Keep local-only mode distinct from any explicitly permitted remote fallback. Browser text composition and Pi planning still use their configured models; replacing JEV alone does not make the entire workflow private or offline.
|
||||
|
||||
## Evidence and limitations
|
||||
|
||||
No model weights were downloaded, no inference was performed, and no production code or configuration was changed. Source was fetched with OpenSrc; hardware was inspected on both hosts. The only executed compatibility probe exercised decider's pure-Python response formatter. Djev's researcher attempted CPU tests but could not run them because `httpx` was absent; no test pass is claimed.
|
||||
|
||||
All candidate benchmark numbers are author-reported. Source snapshots live under `/srv/data/opensrc/repos/github.com/`. OpenSrc omits Git metadata; the cited upstream commits were identified during research, but byte-for-byte snapshot identity was not independently verified for decider/djev. Pin and verify exact artifacts before a trial.
|
||||
|
||||
Licenses inspected: SemIf code MIT; decider/djev code Apache-2.0; the cited Qwen, MiniCPM5, and DiffusionGemma weight cards declare Apache-2.0. Preserve notices and separately check any chosen third-party quantization/runtime artifacts. An open weight license does not establish training-data provenance or decision quality. [2–4]
|
||||
|
||||
## Sources
|
||||
|
||||
1. Local integration: [`jev.ts`](../extensions/fast-browser/jev.ts), [`contracts.ts`](../extensions/fast-browser/contracts.ts), [`controller.ts`](../extensions/fast-browser/controller.ts), [`scenario-cli.ts`](../extensions/fast-browser/scenario-cli.ts).
|
||||
2. Decider source at [`c4daaac`](https://github.com/Mapika/decider/tree/c4daaac28af9fea95d627015cffa2dd5a5926ee6): [README/results](https://github.com/Mapika/decider/blob/c4daaac28af9fea95d627015cffa2dd5a5926ee6/README.md), [response formatter](https://github.com/Mapika/decider/blob/c4daaac28af9fea95d627015cffa2dd5a5926ee6/decider/systemone.py#L119-L130), [server defaults](https://github.com/Mapika/decider/blob/c4daaac28af9fea95d627015cffa2dd5a5926ee6/decider/serve.py#L18-L25); pinned [2B card](https://huggingface.co/Mapika/decider-2b/blob/b37f7e1ba3fbc9238004cf531fabbee2619973fd/README.md), [0.8B card](https://huggingface.co/Mapika/decider-0.8b/blob/1ea54127d3bd52f6d753d9257b32a6380b873907/README.md), [35B NVFP4 card](https://huggingface.co/Mapika/decider-35b-a3b-nvfp4/blob/080221dad5a9184dc5edbba26909089abea42d73/README.md).
|
||||
3. SemIf at [`ca3ba65`](https://github.com/TheoLeeCJ/SemIf/tree/ca3ba65f142967030ecb453346e94d6f476a69df): [results and limitations](https://github.com/TheoLeeCJ/SemIf/blob/ca3ba65f142967030ecb453346e94d6f476a69df/docs/RESULTS.md), [Python limits](https://github.com/TheoLeeCJ/SemIf/blob/ca3ba65f142967030ecb453346e94d6f476a69df/src/semif_phase1/core.py), [browser worker](https://github.com/TheoLeeCJ/SemIf/blob/ca3ba65f142967030ecb453346e94d6f476a69df/webgpu-demo/worker.js), [memory measurements](https://github.com/TheoLeeCJ/SemIf/blob/ca3ba65f142967030ecb453346e94d6f476a69df/results/phase1-summary.json), [artifact/license inventory](https://github.com/TheoLeeCJ/SemIf/blob/ca3ba65f142967030ecb453346e94d6f476a69df/THIRD_PARTY.md).
|
||||
4. Djev at [`3ce907e`](https://github.com/Davipar/djev-dev/tree/3ce907e6835212f27ee82b4cee9039198c4abe35): [runtime](https://github.com/Davipar/djev-dev/blob/3ce907e6835212f27ee82b4cee9039198c4abe35/docs/runtime.md), [performance caveats](https://github.com/Davipar/djev-dev/blob/3ce907e6835212f27ee82b4cee9039198c4abe35/docs/performance.md), [confidence contract](https://github.com/Davipar/djev-dev/blob/3ce907e6835212f27ee82b4cee9039198c4abe35/djev/contracts.py#L225-L271); [pinned DiffusionGemma card](https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md).
|
||||
5. TypeSafe: [current pricing/limits](https://docs.typesafe.ai/models), [latency claims](https://typesafe.ai/blog/introducing-system-one-models-and-jev), [parallel questions](https://docs.typesafe.ai/cookbooks/parallel_questions), [privacy policy](https://typesafe.ai/legal/privacy-policy). No-training claims do not imply zero retention.
|
||||
6. Hosting: [Vast 4090 pricing](https://vast.ai/pricing/gpu/RTX-4090), [Vast billing](https://docs.vast.ai/guides/reference/billing), [RunPod pricing](https://www.runpod.io/pricing), [serverless operation](https://docs.runpod.io/serverless/overview).
|
||||
7. Historical local evidence: [fast-browser demo](fast-browser-live-demo.md), [prototype validation](fast-browser-prototype-validation.md), [original JEV feasibility](browser-use-jev-feasibility.md).
|
||||
@@ -155,7 +155,7 @@ Slices in the same row may proceed in parallel after their listed dependencies a
|
||||
- one selected provider route;
|
||||
- no package registries, SSH, Docker, browser, microphone, deployment credentials, or broad localhost access.
|
||||
4. Separate extension source from extension state. Confirm that normal sessions cannot modify global instructions, installed extension code, authentication state, other Pi profiles, or sibling repositories.
|
||||
5. Materialize the normal in-process extension allowlist as a generated, isolated Pi `settings.json` under the selected personal/work state root. Its package/extension entries come only from `config/pi-package-lock.json` and the trusted repository map. Disable `pi-ask` arbitrary notification commands, `/update`, `workflow`, background shell jobs, and any privileged direct spawn path until a controlled subprocess policy exists.
|
||||
5. Materialize the normal in-process extension allowlist as a generated, isolated Pi `settings.json` under the selected personal/work state root. Its package/extension entries come only from `config/pi-package-lock.json` and the trusted repository map. Disable `pi-ask` arbitrary notification commands, `/update`, the legacy `workflow` extension, background shell jobs, and any privileged direct spawn path until a controlled subprocess policy exists. The `develop-workflow` skill adds no executable or permission boundary; it composes only tools already available to the parent session.
|
||||
6. Import the unmanaged `~/.pi/agent/extensions/lm-studio.ts` implementation into a repository-managed extension at `extensions/lm-studio/index.ts`, lock its source revision, and change its fixed base URL contract to `http://127.0.0.1:1234/v1`. Maintenance moves the old extension to the quarantined state store and removes its active settings entry. A duplicate-provider test fails if the old source, another `lm-studio` registration, or `localhost` base URL remains loadable.
|
||||
7. Define and own stable profile fragments for `provider-lm-studio`, `provider-openai`, `provider-anthropic`, and `provider-morph`, granting only one route at launch. Prove nono 0.74's `tcp_connect_ports` behavior with real loopback fixtures listening on ports 1234 and 1235: the selected provider must connect to 1234 while 1235 and every other loopback port fail. Do not use hostname-only `allow_domain` as evidence of port restriction. If the fixture cannot prove exact enforcement, `provider-lm-studio` remains unavailable until a separately reviewed mediated relay is designed.
|
||||
|
||||
@@ -165,7 +165,7 @@ Slices in the same row may proceed in parallel after their listed dependencies a
|
||||
- The selected provider works while non-selected provider hosts, raw key environment names, and credentials are absent from the environment and process table.
|
||||
- Reads outside the exact Pi state, declared active-worktree access, and per-launch temporary paths fail; writes to a read-only active worktree fail; another session's temporary directory is inaccessible.
|
||||
- LM Studio, when selected, reaches only `127.0.0.1:1234`.
|
||||
- `/update`, `workflow`, arbitrary notification commands, and unbrokered background shell jobs are unavailable.
|
||||
- `/update`, the legacy `workflow` extension, arbitrary notification commands, and unbrokered background shell jobs are unavailable; skill-led development cannot widen the parent session's policy.
|
||||
- The repository-managed LM Studio provider uses only the exact `127.0.0.1:1234` base URL, the old unmanaged source is quarantined, duplicate registration fails, and a real two-port fixture proves all other loopback ports are denied.
|
||||
- All base and provider profiles pass strict validation and resolved-policy review.
|
||||
|
||||
@@ -381,32 +381,34 @@ A failed pilot returns to the owning slice. Do not compensate by broadening the
|
||||
|
||||
## Slice 6B1: Child-agent and background-task isolation
|
||||
|
||||
**Goal:** isolate spawned Pi, Claude, Codex, bounded background jobs, Herdr panes, and tmux panes without enabling repository toolchain execution or `workflow`.
|
||||
**Goal:** isolate spawned Pi, Claude, Codex, bounded background jobs, Herdr panes, and tmux panes without enabling repository toolchain execution or the legacy `workflow` extension.
|
||||
|
||||
**Owner surfaces:** `profiles/nono/{child-pi,child-claude,child-codex,background-jobs}.json`, new spawn-wrapper sources under `scripts/nono/child-agents/`, isolated child-state configuration, and `tests/nono/capabilities/child-agents/`. Coordinate with extension owners but avoid editing Slice 6A's socket fragments or shared extension files concurrently.
|
||||
|
||||
**Work:**
|
||||
|
||||
1. Inventory every spawn path in active subagent, `pi-ask`, update, and background-task extensions. Classify each in-process operation as safe UI/state access, disabled behavior, or a brokered subprocess. Keep `workflow` absent from generated settings.
|
||||
1. Inventory every spawn path in active subagent, `pi-ask`, update, and background-task extensions. Classify each in-process operation as safe UI/state access, disabled behavior, or a brokered subprocess. Keep the legacy `workflow` extension absent from generated settings.
|
||||
2. Route child Pi, Claude, and Codex through command policies with isolated child state and the immutable provider selected by Slice 3.
|
||||
3. Broker only named background jobs and notification commands that do not invoke repository toolchains; keep arbitrary commands, package updates, and direct privileged spawn paths disabled.
|
||||
4. Define caller relationships and prohibit privileged-command escalation through child commands.
|
||||
5. Verify Herdr and tmux targets launch under equivalent or narrower generated policies before enabling command delivery.
|
||||
6. Add containment tests for nested children, background completion, cancellation, inherited environment, provider override denial, notification commands, package-update denial, and artifact paths.
|
||||
|
||||
**Acceptance criteria:** child commands receive no broader filesystem, network, credential, socket, environment, or provider access than declared; nested child tests prove containment; unsandboxed Herdr/tmux targets are rejected; bounded extension behavior remains functional without `workflow` or toolchain execution.
|
||||
**Acceptance criteria:** child commands receive no broader filesystem, network, credential, socket, environment, or provider access than declared; nested child tests prove containment; unsandboxed Herdr/tmux targets are rejected; bounded extension behavior remains functional without the legacy `workflow` extension or toolchain execution.
|
||||
|
||||
**Dependencies:** pilot acceptance. May run in parallel with Slices 5A, 5B, and 6A.
|
||||
|
||||
## Slice 6B2: `workflow` and toolchain integration
|
||||
## Slice 6B2: Skill-led development toolchain evidence
|
||||
|
||||
**Goal:** enable `workflow` only after every toolchain command it can invoke has a completed command policy.
|
||||
> [ADR 0006](adr/0006-skill-led-development-workflow-04SEP2026.md) supersedes the planned `workflow` extension profile, spawn wrapper, generated-settings entry, and capability suite. Do not create those surfaces.
|
||||
|
||||
**Owner surfaces:** `profiles/nono/workflow.json`, workflow spawn-wrapper sources under `scripts/nono/child-agents/workflow/`, the generated extension allowlist entry, and `tests/nono/capabilities/workflow/`.
|
||||
**Goal:** prove that `develop-workflow` uses only the parent session's approved Git, package-manager, verification, and child-agent command policies.
|
||||
|
||||
**Work:** route `workflow` executable resolution and every npm/pnpm/Bun/Python/Rust/Go/Maven/Gradle task through the completed Slice 5 policies. Include lifecycle/run scripts, nested package executables, verification commands, and cancellation. Any unmodeled executable or repository without the required toolchain fragment fails before workflow execution.
|
||||
**Owner surfaces:** evidence under the existing Slice 5 and Slice 6B1 suites. The skill itself owns no nono profile, executable, wrapper, extension entry, or permission grant.
|
||||
|
||||
**Acceptance criteria:** adversarial workflow tasks cannot bypass toolchain policies, select another provider, invoke high-risk commands, read sibling repositories, or access raw credentials; repositories lacking a required toolchain capability do not load `workflow`.
|
||||
**Work:** exercise representative skill-led assignments through the already completed toolchain and child-agent policies. Verify that model-selected commands cannot bypass executable allowlists, lifecycle/run-script containment, provider binding, repository boundaries, or cancellation. An unmodeled executable or repository without the required fragment fails exactly as it does for direct parent tool use.
|
||||
|
||||
**Acceptance criteria:** skill-led tasks cannot broaden parent or child policy, select another provider, invoke high-risk commands, read sibling repositories, or access raw credentials. The legacy `workflow` extension remains absent from generated settings.
|
||||
|
||||
**Dependencies:** Slices 5A, all applicable 5B language fragments, and Slice 6B1. It does not run in parallel with the toolchain policy it consumes.
|
||||
|
||||
@@ -479,14 +481,14 @@ To keep parallel work mergeable:
|
||||
| Base filesystem | Active registered worktree has exactly its mapped read-only or read/write access; exact mutable Pi state and the per-launch temporary directory are writable | A read-only worktree cannot be written; broad `/tmp`, runtime parents, other sessions, sibling or unregistered worktrees, global instructions, extension source, other Pi state, `~/.ssh`, and `~/.config/gh` are inaccessible or read-only as designed |
|
||||
| Temporary lifecycle | Normal exit and cancellation remove the private runtime directory; approved active use remains within its aggregate bound | Cleanup preserves a held lease, matching PID/start time, populated cgroup, live unit, or inconclusive identity; proven crash leftovers expire, unsafe paths survive for review, and an active session cannot exhaust `/run` |
|
||||
| Worktree lifecycle | Broker creates, registers, resolves, and safely removes an external clean worktree | Broad worktree-root access, age-only deletion, dirty/untracked/unpushed removal, active-worktree removal, and linked-worktree confidentiality claims fail |
|
||||
| Environment/extensions | Reviewed extensions and selected mediated provider credential work | Raw credential variables, old LM Studio source, duplicate providers, `/update`, `workflow` before 6B2, arbitrary notification commands, and unbrokered shell jobs are absent |
|
||||
| Environment/extensions | Reviewed extensions and selected mediated provider credential work | Raw credential variables, old LM Studio source, duplicate providers, `/update`, the legacy `workflow` extension, arbitrary notification commands, and unbrokered shell jobs are absent |
|
||||
| Provider | Trusted default or one pre-boundary approved flag selects the route; LM Studio reaches port 1234 | Ambiguous/post-boundary/child overrides, non-selected providers, LM Studio port 1235, and broad localhost fail |
|
||||
| Git/SSH | Fetch/push uses the correct provisioned agent and immutable wrapper | Direct SSH, stale/substituted agent, bypass config, alternate binary, same-port host, wrong agent, wrong host, and wrong port fail |
|
||||
| GitHub CLI | Active-repository operation succeeds through proxy | Real token exposure, cross-repository API, and unapproved endpoints fail |
|
||||
| Package manager/workflow | Restore/build/test uses exact cache and hosts; 6B2 routes workflow tasks through completed policies | Lifecycle/run-script escape, unrelated cache, publish token, privileged command, and unapproved registry fail |
|
||||
| Package manager/skill-led tasks | Restore/build/test uses exact cache and hosts; 6B2 proves `develop-workflow` cannot exceed completed parent policies | Lifecycle/run-script escape, unrelated cache, publish token, privileged command, and unapproved registry fail |
|
||||
| Orca/Herdr | Authenticated current endpoint and verified socket peer succeed on launch and cache hit | Stale/replaced/wrong-service endpoint and unrelated socket/port fail |
|
||||
| Dictation | Voice request succeeds with `--voice` | Socket, audio stack, model data, and GPU remain absent by default |
|
||||
| Child agents | Declared bounded child workflow completes | Child cannot broaden filesystem, network, credentials, sockets, environment, or provider |
|
||||
| Child agents | Declared bounded child task completes | Child cannot broaden filesystem, network, credentials, sockets, environment, or provider |
|
||||
| High-risk modes | Explicit host-authorized workflow succeeds and records policy digest | Missing confirmation, conflicting flags, sandbox invocation, or another mode's capability fails |
|
||||
| Generator/evidence | Strict validation, dynamic revalidation, deterministic cache hit, and trusted evidence summary succeed | Unknown or malformed inputs, stale digests/endpoints, tampered cache/evidence producer, and unsafe runtime/cache paths stop before launch |
|
||||
|
||||
|
||||
@@ -20,9 +20,9 @@ Captured for Slice 0 on 27AUG2026 with nono 0.74.0. This record contains no cred
|
||||
| `pi-orca-local` | Symlink to `profiles/nono/pi-orca-local.json` | Approved drift imported, narrowed to read-only trusted-file access, and relinked on 27AUG2026. |
|
||||
| `pi-harness-tools` | Symlink to `profiles/nono/pi-harness-tools.json` | Approved drift imported, narrowed to read-only trusted-file access, and relinked on 27AUG2026. |
|
||||
|
||||
The imported `pi-orca-local` drift is limited to exact read-only access for `$HOME/.pi/agent/AGENTS.md`. Its saved grant for `$HOME/.pi/agent/extensions/workflow/index.ts` was not imported because the user removed that extension and deferred its replacement to a future skill. The imported `pi-harness-tools` drift is limited to exact read-only access for `$HOME/.pi/agent/extensions/pi-todo-herdr/index.ts` and `$HOME/.config/herdr/herdr.sock`. Its inherited `pi-local` policy already contains the Herdr Unix-socket grant. The active files used read/write `allow_file` saved grants; the repository deliberately narrows them to `read_file` because normal sessions do not need to modify trusted instructions or extension code. No other active saved grant was imported.
|
||||
The imported `pi-orca-local` drift is limited to exact read-only access for `$HOME/.pi/agent/AGENTS.md`. Its saved grant for `$HOME/.pi/agent/extensions/workflow/index.ts` was not imported because the user removed that extension; [ADR 0006](adr/0006-skill-led-development-workflow-04SEP2026.md) replaces it with the `develop-workflow` skill, which needs no extension-source grant. The imported `pi-harness-tools` drift is limited to exact read-only access for `$HOME/.pi/agent/extensions/pi-todo-herdr/index.ts` and `$HOME/.config/herdr/herdr.sock`. Its inherited `pi-local` policy already contains the Herdr Unix-socket grant. The active files used read/write `allow_file` saved grants; the repository deliberately narrows them to `read_file` because normal sessions do not need to modify trusted instructions or extension code. No other active saved grant was imported.
|
||||
|
||||
The `workflow` extension was unlinked from active Pi state by the user during this slice. It is also absent from normal `config/pi-links.json` exposure so reconciliation will not recreate it. The future replacement is tracked in `docs/todos.md`.
|
||||
The legacy `workflow` extension was unlinked from active Pi state by the user during this slice. It remains absent from normal `config/pi-links.json` exposure so reconciliation will not recreate it. The `develop-workflow` skill replacement and its remaining pilot are tracked in `docs/todos.md`.
|
||||
|
||||
## Draft disposition
|
||||
|
||||
|
||||
@@ -42,18 +42,21 @@ bun publish
|
||||
|
||||
Publish `@jbcode/pi-shared` before a new version of any extension that depends on it. Gitea rejects republishing an existing name/version, so bump the changed package version first.
|
||||
|
||||
`extensions/workflow` is intentionally private until the `_workflows` reference dependencies are merged into that package. `extensions/_workflows` must not be published.
|
||||
`__archived/extensions/workflow` and `__archived/extensions/_workflows` are historical reference, outside active workspaces, and must not be published. [ADR 0007](adr/0007-archive-legacy-workflows-05SEP2026.md) records their archival and supersedes ADR 0006's pilot-before-retirement prerequisite; new development uses the `develop-workflow` skill.
|
||||
|
||||
## Install with Pi
|
||||
|
||||
After configuring the same scope and token in the npm configuration visible to Pi, install the currently deployed release set with exact version pins:
|
||||
|
||||
```bash
|
||||
pi install npm:@jbcode/pi-dictate@0.1.0
|
||||
pi install npm:@jbcode/pi-fast-browser@0.1.1
|
||||
pi install npm:@jbcode/pi-dictate@0.2.0
|
||||
pi install npm:@jbcode/pi-effort-controls@0.1.0
|
||||
pi install npm:@jbcode/pi-prompt-snippets@0.1.0
|
||||
pi install npm:@jbcode/pi-interactive-subagents@3.7.3
|
||||
pi install npm:@jbcode/pi-interactive-subagents@3.7.4
|
||||
pi install npm:@jbcode/pi-todo-herdr@0.2.5
|
||||
```
|
||||
|
||||
Pi installs runtime dependencies from the same scoped registry. `config/pi-links.json` deliberately exposes no extension symlinks, preventing duplicate local and published copies.
|
||||
|
||||
`@jbcode/pi-bash-cwd` is currently source-only and is not part of the deployed release set above. It can be packed and published through the same workspace process when a Pi 0.84.x installation needs the alternate-cwd compatibility override.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Explicit repository relocation
|
||||
|
||||
The link manager normally rejects symlinks outside the current checkout. After a repository move, explicitly authorize the previous root rather than weakening that default:
|
||||
|
||||
```bash
|
||||
node scripts/pi-links.mjs sync --dry-run --migrate-from /absolute/old/root
|
||||
node scripts/pi-links.mjs sync --migrate-from /absolute/old/root
|
||||
node scripts/pi-links.mjs check
|
||||
```
|
||||
|
||||
Inspect the dry run before applying. Migration accepts only a separate absolute old root and only repairs manifest-declared destinations whose resolved link target exactly equals the old root plus the current source's repository-relative path. Dangling old targets are supported. Regular files, directories, unrelated targets, and mismatched relative paths remain conflicts. Replacement rechecks the target immediately before unlinking; this is not an atomic filesystem compare-and-swap.
|
||||
|
||||
The option does not claim stale/unlisted links or old local package registrations. It is rejected for `prune`; normal pruning still requires its own preview and approval. Linked-worktree ownership is a separate issue, not solved by this relocation option.
|
||||
|
||||
## Validated host migration
|
||||
|
||||
Moved exposure from `/srv/data/code/jb-pi/harness-tools` to `/home/jb/code/harness-tools`: 38 global links repaired (35 skills and three instruction files), two project-local skill links created. Normal `check` passed with 40 links and zero local package entries. A bounded scan of harness configuration directories found no remaining symlinks with that old absolute target prefix. All 12 link-manager tests passed, including relocation, dry-run, collision preservation, invalid-input, and no-prune coverage.
|
||||
|
||||
Browser routing was also updated in `skills/tools/browser-harness/SKILL.md`: fast-browser is the default for supported local tests, while browser-harness requires explicit opt-in and must never bypass failed actions or share a fast-browser-owned session. Run `/reload` to refresh skill exposure.
|
||||
@@ -0,0 +1,142 @@
|
||||
# Fast-browser performance workflow record
|
||||
|
||||
## Status
|
||||
|
||||
- Status: complete
|
||||
- Current phase: verified, independently approved product fast-forwarded to local main.
|
||||
- Next safe action: after this record-only closure commit, run final sanity/reachability checks, remove the owned clean worktree/branch, and restore the unrelated file's Git visibility. Final housekeeping evidence is written to `/tmp/pi-present/fast-browser-performance-cleanup.json`.
|
||||
|
||||
## Request and scope
|
||||
|
||||
Optimize fast-browser for local development testing and validation, borrowing proven round-trip reductions from jev-ultrafast. The parent orchestrates; subagents implement exclusive slices. Optimize browser/runtime overhead first, with measured correctness-preserving gains rather than a speculative autonomous-agent rewrite.
|
||||
|
||||
In scope: redundant browser calls; AX-preserving batched observation metadata; local origin-context calculation; a provider-free paired benchmark; focused regression tests and performance documentation.
|
||||
|
||||
Out of scope: new actions, autonomous planning, broader model context, changed model/provider, dependency changes, public API changes, uploads/downloads, personal browser profiles, production access, publishing, and speculative page-state/decision caches. No product UI change. Synthetic fixture HTML is validation material, not a new UI feature.
|
||||
|
||||
## Git identity
|
||||
|
||||
- Source branch: `main` at `773c628512fdfdf5a139ae1b27833bea97d74255`.
|
||||
- Remote: `origin/main`, fetched and identical to local main.
|
||||
- Feature branch: `feature/fast-browser-performance` (must be absent before creation).
|
||||
- Worktree: `/home/jb/code/harness-tools/.worktrees/fast-browser-performance` (must be absent before creation).
|
||||
- Omitted source changes: untracked `docs/local-decision-model-feasibility.md`, preserved untouched. No tracked source modifications observed.
|
||||
- Add `/.worktrees/` to the repository-local Git exclude file before worktree creation. This is local Git housekeeping, not a product change.
|
||||
- Record path after approval: `docs/plans/fast-browser-performance.md` in the new worktree.
|
||||
- Reuse existing root node_modules and the pinned Python environment `/home/jb/code/harness-tools/.pi/fast-browser-demo/venv/bin/python`; no installation or lockfile modification. Any worktree-local links are ignored and explicitly owned.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
1. Existing public tools, operation schemas, candidate ordering/limit, field-label privacy, fresh action revalidation, occlusion checks, outcome verification, budgets, session isolation, and no-blind-retry semantics remain intact.
|
||||
2. Browser hot-path call counts fall deterministically: one target inventory per refresh; no redundant domain enables after the pinned harness attach; browser platform detection at most once per backend; batched observations remove per-candidate partial-AX queries and repeated evaluate/release cycles.
|
||||
3. Chrome AX data remains authoritative for observation roles and click names. Fill labels retain the existing value-free DOM extraction and never fall back to AX value-bearing names. No field values, raw DOM, selectors, or arbitrary scripts enter observations, provider input, or benchmark reports.
|
||||
4. Batch resources exist only for one bounded observation chunk and are released on success/failure. Fresh single-target metadata remains the pre-mutation authority. Missing/failed batch data uses the existing safe per-node path where compatible; no blind mutation retry.
|
||||
5. Origin narrowing remains validated and session-lifetime/serialization aware but does not send a redundant JSONL operation. The effective context still travels with every subsequent browser operation; omitted context restores the session base, and restricted sessions cannot broaden.
|
||||
6. Benchmark results cover unchanged baseline and optimized code with the same runner/fixture/browser, independent outcome assertions, all attempts retained, startup separate from warm operation timing, p50/p95 and protocol counts. Keep optimizations only when call counts improve and live results show no material regression; investigate noisy or mixed results before claiming gains.
|
||||
7. Full repository tests/typecheck, focused regressions, real Chromium runtime validation, and two independent review axes pass for the final committed diff. Any environment blocker or unsupported claim remains explicit.
|
||||
|
||||
## Approved implementation plan and gates
|
||||
|
||||
Plan approval: user selected `approve` (Approve and implement) through ask_user on 2026-09-21. Approval covers the exact slices and source-file omission below, local benchmark browsers/fixtures, local commits, and eventual safe fast-forward integration. It does not authorize dependency changes, production access, API/security-posture changes, or publication.
|
||||
|
||||
### Slice A: provider-free measurement and regression fixture
|
||||
|
||||
Owner `performance-benchmark` (worker). Exclusive paths: `extensions/fast-browser/performance-benchmark.ts`, `extensions/fast-browser/performance-benchmark.test.ts`, `extensions/fast-browser/performance_probe.py`, `extensions/fast-browser/fixtures/performance.html`.
|
||||
|
||||
Build a bounded opt-in runner using existing browser-session/runtime/transport seams. Accept explicit worker/baseline paths and pinned Python/browser configuration rather than copying an old implementation into production. Count CDP/IPC methods and JSONL operations locally without recording arguments, URLs, values, text, or screenshots. A test-only Python probe may wrap the selected worker's helper methods for counting; no product instrumentation switch is required.
|
||||
|
||||
Use realistic local account/settings or search/filter form workflows with 0/10/60 eligible controls, fill replacement, immediate/delayed click outcomes, and popup discovery. Assert actual final state independently of action postconditions. Exercise warm repeated observations and complete verified flows; measure cold startup separately. Alternate baseline/optimized arms, aim for at least 20 paired warm samples, retain failures and cleanup results. Deterministic model stubs may exercise orchestration without provider cost; clearly exclude real provider latency and semantic accuracy from claims. Focused tests cover statistics/report accounting, input bounds, failure inclusion, and report privacy. No wall-clock thresholds in unit tests.
|
||||
|
||||
### Slice B: remove proven browser-call duplication
|
||||
|
||||
Owner `browser-hotpath` (worker). Exclusive paths: `extensions/fast-browser/browser_worker.py`, `extensions/fast-browser/test_browser_worker.py`.
|
||||
|
||||
Use a single `Target.getTargets` response for page inventory and opener metadata, preserving filtering and target identity. Remove duplicate Page/DOM/Runtime/Network enables only where pinned browser-harness 0.1.13 already guarantees attempts on attachment. Cache browser-version-derived select-all modifiers per backend lifetime. Reuse the already obtained tab result during fill URL verification. Add exact call-count and behavioral regressions. Do not lengthen polling intervals as a speculative throughput optimization or cache changing document/target state across operations.
|
||||
|
||||
### Slice C: AX-preserving observation batching
|
||||
|
||||
Owner `browser-hotpath`, serialized after B, same exclusive paths. Use an internal Python-owned script constant to avoid new runtime assets and packaging changes.
|
||||
|
||||
Introduce a private optional batch-metadata seam with the legacy backend/fake fallback. Keep full AX candidate discovery and ordering. Resolve eligible backend node IDs into a bounded temporary object group; evaluate existing value-free DOM metadata for a small chunk together, using full AX snapshot role/name rather than redundant partial AX reads; release the object group in finally. Never retain remote node handles or geometry across observations. Keep fresh existing single-node AX/DOM revalidation before act. Cover distinct AX/DOM labels, password/value privacy, disabled/hidden/disconnected nodes, duplicate candidates, limits, malformed/partial responses, navigation/detachment and cleanup. Measure against the baseline before retaining. If the compatible batch path cannot demonstrate safe benefit, retain B and document the blocked experiment instead of weakening contracts.
|
||||
|
||||
### Slice D: local origin-context calculation
|
||||
|
||||
Owner `runtime-hotpath` (worker). Exclusive paths: `extensions/fast-browser/runtime.ts`, `extensions/fast-browser/runtime.test.ts`.
|
||||
|
||||
Compute `narrowOriginPolicy` locally, retaining runtime closed/aborted behavior, serialized session semantics and current validation/error shaping. Subsequent transport/worker operations must still enforce the returned context. Preserve compatibility for direct legacy transport requests. Test no transport dispatch for narrowing, context propagation, rejected broadening, base restoration, closed/aborted sessions and concurrency behavior. Do not remove transport validation, serialized lanes, or change error/mutation semantics to save time.
|
||||
|
||||
### Orchestration and documentation
|
||||
|
||||
Parent owns `docs/plans/fast-browser-performance.md`, `docs/fast-browser-performance.md`, and the performance section of `extensions/fast-browser/README.md`. No code implementation is assigned to the parent. Record measured boundaries, exact commands, baseline identity, limitations and retained results. Add `docs/todos.md` evidence only for a newly encountered tool defect that cannot be resolved in scope.
|
||||
|
||||
Each mutating assignment has a 15-minute deadline. B and C are separate assignments to the same owner; A and D may run in parallel because paths are disjoint. Workers do not commit, delegate mutation, alter dependencies, modify other owners' files, or integrate branches. Parent records before/after Git evidence and inspects every returned diff. A worker hitting its deadline must stop and report its partial state; reassignment requires proven completion.
|
||||
|
||||
## Ownership
|
||||
|
||||
| Assignment | Owner identity | Deadline | Exclusive paths | Before/after Git evidence | Outcome |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Research browser | browser-performance-research / scout | complete | none; read-only | source unchanged | delivered |
|
||||
| Research runtime | runtime-performance-research / scout | complete | none; read-only | source unchanged | delivered |
|
||||
| A benchmark | performance-benchmark / worker | first and correction deliveries within deadlines | four listed new files | initial owned patch `/tmp/pi-present/performance-slice-a-initial.patch`; parent inspected corrections and reran 6 tests/compile/check | committed e10a957; stable 20-pair measurement started |
|
||||
| B browser duplication | browser-hotpath / worker | completed within deadline | worker Python + Python tests | delivered only owned paths; `/tmp/pi-present/performance-slice-b.patch`; parent reran 47 Python tests (pass) and diff check | committed a79418e |
|
||||
| C observation batching | browser-hotpath / worker | delivered within deadline | same Python files | owned patch `/tmp/pi-present/performance-slice-c.patch`; parent reran 53 Python tests and diff check | committed 574e083; optimized real Chromium runtime validation passed |
|
||||
| D origin context | runtime-hotpath / worker | completed within deadline | runtime.ts + runtime.test.ts | owned patch `/tmp/pi-present/performance-slice-d.patch`; parent reran 12 tests and typecheck | committed c0c18a5 |
|
||||
| Docs/integration | parent | phase bounded | listed docs/README only | record checkpoints | pending |
|
||||
|
||||
## Verification matrix
|
||||
|
||||
| Surface | Command | Exit code | Outcome |
|
||||
| --- | --- | --- | --- |
|
||||
| Baseline typecheck | `bun run typecheck` | 0 | Passed; an accidental nonexistent chained test command then failed, unrelated to repository correctness. |
|
||||
| Baseline full tests | `bun run test` | 0 | Passed. |
|
||||
| Baseline fast-browser | `bun run test:fast-browser` | 0 | 220 TS passed, 2 skipped; 42 Python passed. |
|
||||
| Focused Python | `.pi/fast-browser-demo/venv/bin/python -m unittest discover -s extensions/fast-browser -p 'test_*.py'` | 0 | 61 passed, including counting, batching, stale/privacy, delayed-close and readiness fake-clock regressions. |
|
||||
| Focused runtime/benchmark | individual `node --test --experimental-strip-types` runs for runtime.test.ts and performance-benchmark.test.ts | 0 | Parent verified 12 runtime and 7 benchmark tests. |
|
||||
| Integrated static/offline | `bun run typecheck && bun run test` | 0 | Full repository suite passed. Fast-browser: 230 TS passed, 2 opt-in skips; 61 Python passed. No dependency installation. |
|
||||
| Scenario validation | scenario-cli `--check --format json` for existing local-form/login/popup examples | 0 | All three validated; no browser started by preflight. |
|
||||
| Real runtime | `FAST_BROWSER_DEVELOPMENT_LIVE=1 node --test --experimental-strip-types extensions/fast-browser/development-live-validation.test.ts` | 0 | Passed with no skip on baseline, batched worker and final corrected worker. Includes private evidence and cleanup. |
|
||||
| Paired benchmark | command in `docs/fast-browser-performance.md`, 20 pairs plus 3 cold samples per arm | 0 | Final all 46 attempts passed, counters complete, cleanup passed. Warm p50 702 to 279ms; p95 729.05 to 315.30ms; CDP 17,791 to 7,407. Earlier failures retained. |
|
||||
| Hygiene/package | `git diff --check`; root package tests in full suite | 0 | Passed. Metadata script is an internal Python constant; no external runtime resource needed. |
|
||||
|
||||
## Review
|
||||
|
||||
- Review base: `773c628512fdfdf5a139ae1b27833bea97d74255`; product HEAD: `54303e0b5d8d99dbd3556a7a97453157e7ed09fd`; product tree: `08de591daedd2b32a64d3e054097d9cdd040089d`.
|
||||
- Exact command: `git diff --binary --full-index --find-renames 773c628512fdfdf5a139ae1b27833bea97d74255...54303e0b5d8d99dbd3556a7a97453157e7ed09fd --`.
|
||||
- Diff SHA-256: `717829461c80874a99ca24f5ef0d85094e237b1b3238ef40c827672f5b0b670e`.
|
||||
- Commit list, newest first: 54303e0, 2dc359c, fe643e2, 12a6bb6, 6a725c1, f2e42f5, acd57ae, e10a957, 574e083, 4360f01, c0c18a5, 3c321b3, a79418e, 05793fe, 6fba628. Reproduce with `git log --oneline <base>..<product-head>` using the pinned full IDs above.
|
||||
- Excluded mutators: parent and all worker sessions (`browser-hotpath`, `runtime-hotpath`, `performance-benchmark`). Stable worker session IDs were not exposed, so the worker profile was excluded.
|
||||
- Standards reviewer: `performance-standards-review`, read-only claim-verifier, session `01a0c515-7d1a-7680-9e5b-d16aa4202b6b`. Confirmed exact digest and APPROVED with zero findings.
|
||||
- Specification reviewer: `performance-specification-review`, independent read-only claim-verifier; stable session ID not exposed. Confirmed exact digest and APPROVED with zero findings.
|
||||
- Both reviewers and parent confirmed unchanged clean HEAD/tree/status before and after review. No remediation cycle was required. Specifications were this approved record and development contracts; standards were repository/global AGENTS.md and established fast-browser patterns.
|
||||
- Full brief and separate axis results are retained at `/tmp/pi-present/fast-browser-performance-review-brief.md` and `/tmp/pi-present/fast-browser-performance-review-results.md`. Closure changes only this record, outside the approved product fingerprint.
|
||||
|
||||
## Integration and rollback
|
||||
|
||||
Source was fetched again and remained at the pinned base; no rebase was needed. Both review approvals and the exact digest were rechecked before `git merge --ff-only feature/fast-browser-performance`, which succeeded on local main. Source main now contains the exact reviewed product.
|
||||
|
||||
The user explicitly approved `temporary_local_exclude` through ask_user for the unrelated untracked `docs/local-decision-model-feasibility.md`: temporarily exclude that exact path, require a clean source checkout, fast-forward, close and clean up, then restore its original untracked visibility. No contents or tracked state may change. The initial and post-fast-forward SHA-256 are both `5bb4fbf4c77742d41a230d25d87c4e4fd607a5bd38851bf61c7eaf022ebfd86c`. Approval evidence is `/tmp/pi-present/fast-browser-integration-approval.md`.
|
||||
|
||||
Product commit is the pinned HEAD above. The next commit touching only this record is the metadata-only closure commit; its ID is discoverable with `git log -1 -- docs/plans/fast-browser-performance.md`. Final sanity checks prove product/closure reachability and clean owned worktree state before normal `git worktree remove` and `git branch -d`. The final cleanup evidence file records actual results after this commit, including restored visibility and unchanged document hash. Only the explicitly owned worktree/branch and temporary local exclude entries are removed. No push, publication, installed-package change or Pi reload occurred.
|
||||
|
||||
Rollback is a normal revert of focused optimization commits, retaining benchmark tooling if useful.
|
||||
|
||||
## Checkpoints and blockers
|
||||
|
||||
- Last committed valid checkpoint: approved plan/assignments at 05793fe; B product a79418e, D product c0c18a5, C product 574e083, A product e10a957. Worktree dependency links include workspace-local node_modules; local Git excludes now ignore these symlinks. No dependency installation occurred.
|
||||
- Blockers: none for implementation. Source still has only the approved unrelated untracked file.
|
||||
- Worktree creation verified at the specified base and path. `/.worktrees/` was already in local Git excludes; no exclude change was needed.
|
||||
- Baseline real Chromium development validation passed (opt-in test exit 0, no skips).
|
||||
- A corrections: alternate pair order, expose per-density timing/count evidence, independently check saved fixture data, make popup postcondition stable, fail/retain evidence on cleanup or missing counters, and sanitize errors to safe categories. This completes existing acceptance criteria without changing public scope.
|
||||
- A correction smoke passed; historical pre-correction transient popup-close failure is preserved at `/tmp/performance-smoke-final.json`, not attributed to concurrent editing. Stable paired measurements must include all failures.
|
||||
- First stable 20-pair result `/tmp/fast-browser-performance-20-pairs.json`: exit 1; baseline 19/20 and optimized 18/20 warm flows passed, all six cold attempts passed, counters/cleanup complete. Failures are popup-close verification (optimized samples 10/13, baseline sample 20). 60-control median observation 79 to 30ms; 10-control 16 to 7ms, but aggregate medians 445 to 551ms. These are diagnostic results, not acceptance.
|
||||
- Additional bounded assignments: browser-hotpath read-only diagnosis (8 minutes), performance-benchmark safe per-operation timing in its existing four paths (15 minutes). Before state acd57ae, product paths clean; only parent performance doc untracked. No scope/API/security expansion: this is required existing benchmark verification.
|
||||
- Browser diagnosis confirmed one-shot close verification races asynchronous Target.closeTarget completion; removing the duplicate inventory makes the pre-existing race more visible. Proposed correction repeats only inventory reads within the existing action-verification budget, never the close mutation. This restores intended verified-close behavior without API/security changes.
|
||||
- Browser-hotpath test-first assignment: 8 minutes, only its existing Python test path, at f2e42f5 with benchmark-owned TS files dirty; product Python remains stable during diagnostic measurement. Add deterministic delayed-disappearance and permanent-presence close cases, prove the first fails before the fix, then return. Before patch `/tmp/pi-present/performance-before-close-tests.patch`.
|
||||
- Timing probe committed 12a6bb6; parent reran 7 benchmark tests and full typecheck. Diagnostic `/tmp/fast-browser-performance-4-pairs-diagnostic-20260921.json` passed all attempts and confirms navigate ~335ms versus observe ~223 to ~88ms. Action times ~110ms remain similar. No provider latency is measured.
|
||||
- Parent reproduced close tests RED: 4 focused tests, 1 expected failure and 1 expected error for delayed/permanent target presence, 2 passed for inventory/cancellation handling. Test patch `/tmp/pi-present/performance-close-red.patch`.
|
||||
- Browser-hotpath correction assignment: 15 minutes, same Python worker/test paths; before HEAD 12a6bb6, only owned red tests dirty plus parent doc. Implement single-dispatch close followed by bounded inventory reads under existing ACTION_TIMEOUT_SECONDS. Replace pinned helper's 300ms readiness sleep with <=25ms bounded monotonic polling for the exact same document.readyState == complete predicate and existing timeout, retaining error propagation. Add readiness regression tests before the fix. This changes private polling cadence, not API, security posture, postconditions or budgets. Before patch `/tmp/pi-present/performance-close-red.patch`.
|
||||
- Corrected close/readiness worker committed 2dc359c; parent inspected patch `/tmp/pi-present/performance-readiness-close-fix.patch` and reran 61 Python tests. Final report `/tmp/fast-browser-performance-20-pairs-final.json` passed all 46 attempts; hash and exact environment/commands are in performance documentation.
|
||||
- Final verification background task bg_vjl_mubjba9s_12 exited 0 after typecheck, complete repository suite, pinned Python suite, real Chromium runtime test, scenario preflight and diff check. No failures; opt-in general-profile test remains skipped in the default suite, while the development live gate was explicitly run.
|
||||
- Both independent reviewers approved the pinned product with zero findings. Source main was clean under the user-approved temporary exact-file exclusion, fast-forward integration succeeded, and the unrelated document hash remained unchanged.
|
||||
- No unresolved product blockers. Failed earlier measurements remain retained and documented.
|
||||
- Next safe action: finalize this record-only closure commit and execute the stated post-commit sanity, reachability, cleanup and visibility-restoration checks.
|
||||
@@ -1,3 +1,5 @@
|
||||
# TODO
|
||||
|
||||
> This item applies only to the private legacy workflow runtime retained during the [`develop-workflow` pilot](adr/0006-skill-led-development-workflow-04SEP2026.md). Mark it obsolete when that runtime is retired unless `GitStateProbe` is intentionally retained.
|
||||
|
||||
- [ ] Make workflow verification tolerate inaccessible sibling Git worktrees. `GitStateProbe.captureRepository()` currently enumerates every path from `git worktree list --porcelain` and fails the verification command before execution when any unrelated worktree cannot be inspected. Preserve detection of cross-worktree mutations without making unrelated worktrees a hard verification dependency. Add focused coverage for an inaccessible registered sibling worktree.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user