feat: stream files as binary frames over WebSocket (#659)

Replace base64-in-JSON file transfers with binary WebSocket frames to
unblock the JS thread when assistant markdown messages contain images.
Receive cost (JSON.parse over multi-MB base64) and persist cost (base64
re-encode to disk) both removed.

- Split daemon-client `exploreFileSystem` into `listDirectory` + `readFile`;
  `readFile` returns `{ bytes, mime, size, modifiedAt }`. Encoding stays
  private to the seam.
- New `packages/server/src/shared/binary-frames/` module hosts terminal
  frames (moved from `terminal-stream-protocol.ts`) and new file-transfer
  opcodes (`FileBegin` / `FileChunk` / `FileEnd`) with request-id correlation.
- Add optional `acceptBinary` to `FileExplorerRequestSchema`. New daemons
  emit binary frames when set; legacy JSON path remains for old clients.
  Compat fallback lives only inside `readFile`.
- New `persistAttachmentFromBytes` writes bytes directly: native via
  `expo-file-system` `File.write(Uint8Array)`, web via Blob -> IndexedDB,
  desktop via new `write_attachment_bytes` IPC. `persistAttachmentFromBase64`
  removed.
This commit is contained in:
Mohamed Boudra
2026-05-02 14:02:27 +08:00
committed by GitHub
parent b6c43088fb
commit bb889dae99
44 changed files with 1630 additions and 436 deletions
@@ -4,6 +4,7 @@ import { createDesktopAttachmentStore } from "./desktop-attachment-store";
const {
copyDesktopAttachmentFileMock,
writeDesktopAttachmentBase64Mock,
writeDesktopAttachmentBytesMock,
deleteDesktopAttachmentFileMock,
garbageCollectDesktopAttachmentFilesMock,
readDesktopFileBase64Mock,
@@ -18,6 +19,10 @@ const {
path: "/managed/att_2.png",
byteSize: 4,
})),
writeDesktopAttachmentBytesMock: vi.fn(async () => ({
path: "/managed/att_bytes.png",
byteSize: 4,
})),
deleteDesktopAttachmentFileMock: vi.fn(async () => true),
garbageCollectDesktopAttachmentFilesMock: vi.fn(async () => 0),
readDesktopFileBase64Mock: vi.fn(async () => "AAECAw=="),
@@ -28,6 +33,7 @@ const {
vi.mock("./desktop-file-commands", () => ({
copyDesktopAttachmentFile: copyDesktopAttachmentFileMock,
writeDesktopAttachmentBase64: writeDesktopAttachmentBase64Mock,
writeDesktopAttachmentBytes: writeDesktopAttachmentBytesMock,
deleteDesktopAttachmentFile: deleteDesktopAttachmentFileMock,
garbageCollectDesktopAttachmentFiles: garbageCollectDesktopAttachmentFilesMock,
}));
@@ -42,6 +48,7 @@ describe("desktop attachment store", () => {
beforeEach(() => {
copyDesktopAttachmentFileMock.mockClear();
writeDesktopAttachmentBase64Mock.mockClear();
writeDesktopAttachmentBytesMock.mockClear();
deleteDesktopAttachmentFileMock.mockClear();
garbageCollectDesktopAttachmentFilesMock.mockClear();
readDesktopFileBase64Mock.mockClear();
@@ -86,28 +93,30 @@ describe("desktop attachment store", () => {
});
});
it("saves raw base64 sources via desktop filesystem writes", async () => {
it("saves raw byte sources via desktop filesystem writes", async () => {
const store = createDesktopAttachmentStore();
const bytes = new Uint8Array([0, 1, 2, 3]);
const attachment = await store.save({
id: "att_base64",
id: "att_bytes",
mimeType: "image/png",
fileName: "inline.png",
source: {
kind: "base64",
base64: "AAECAw==",
kind: "bytes",
bytes,
},
});
expect(writeDesktopAttachmentBase64Mock).toHaveBeenCalledWith({
attachmentId: "att_base64",
base64: "AAECAw==",
expect(writeDesktopAttachmentBytesMock).toHaveBeenCalledWith({
attachmentId: "att_bytes",
bytes,
extension: ".png",
});
expect(writeDesktopAttachmentBase64Mock).not.toHaveBeenCalled();
expect(attachment).toMatchObject({
id: "att_base64",
id: "att_bytes",
mimeType: "image/png",
storageType: "desktop-file",
storageKey: "/managed/att_2.png",
storageKey: "/managed/att_bytes.png",
fileName: "inline.png",
byteSize: 4,
});
@@ -12,6 +12,7 @@ import {
deleteDesktopAttachmentFile,
garbageCollectDesktopAttachmentFiles,
writeDesktopAttachmentBase64,
writeDesktopAttachmentBytes,
} from "@/desktop/attachments/desktop-file-commands";
import {
readDesktopFileBase64,
@@ -132,6 +133,32 @@ async function saveDesktopAttachmentFromBase64(input: {
});
}
async function saveDesktopAttachmentFromBytes(input: {
id: string;
bytes: Uint8Array;
mimeType: string;
fileName?: string | null;
}): Promise<AttachmentMetadata> {
const extension = extensionForAttachment({
fileName: input.fileName,
mimeType: input.mimeType,
});
const result = await writeDesktopAttachmentBytes({
attachmentId: input.id,
bytes: input.bytes,
extension,
});
return toDesktopMetadata({
id: input.id,
mimeType: input.mimeType,
storagePath: result.path,
byteSize: result.byteSize,
fileName: input.fileName,
});
}
function assertDesktopAttachment(attachment: AttachmentMetadata): void {
if (attachment.storageType !== "desktop-file") {
throw new Error(`Unsupported desktop attachment storage type '${attachment.storageType}'.`);
@@ -166,11 +193,11 @@ export function createDesktopAttachmentStore(): AttachmentStore {
});
}
if (input.source.kind === "base64") {
if (input.source.kind === "bytes") {
const mimeType = normalizeMimeType(input.mimeType);
return await saveDesktopAttachmentFromBase64({
return await saveDesktopAttachmentFromBytes({
id,
base64: input.source.base64,
bytes: input.source.bytes,
mimeType,
fileName,
});
@@ -17,6 +17,18 @@ export async function writeDesktopAttachmentBase64(input: {
});
}
export async function writeDesktopAttachmentBytes(input: {
attachmentId: string;
bytes: Uint8Array;
extension?: string | null;
}): Promise<AttachmentFileResult> {
return await invokeDesktopCommand<AttachmentFileResult>("write_attachment_bytes", {
attachmentId: input.attachmentId,
bytes: input.bytes,
extension: input.extension ?? null,
});
}
export async function copyDesktopAttachmentFile(input: {
attachmentId: string;
sourcePath: string;
@@ -1,14 +1,10 @@
import type { AttachmentMetadata } from "@/attachments/types";
import { fileUriToPath } from "@/attachments/utils";
import { invokeDesktopCommand } from "@/desktop/electron/invoke";
import { Buffer } from "buffer";
function base64ToUint8Array(base64: string): Uint8Array {
const binary = atob(base64);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i += 1) {
bytes[i] = binary.charCodeAt(i);
}
return bytes;
return new Uint8Array(Buffer.from(base64, "base64"));
}
const activeObjectUrls = new Set<string>();