fix(server): preserve fatal log on crash and guard socket.send against close races (#613)
The daemon was dying silently with no log trail. Two compounding causes: 1. uncaughtException/unhandledRejection handlers in index.ts called process.exit(1) immediately after logger.fatal, but pino is configured with async streams (sync: false console + rotating-file-stream), so the fatal entry never flushed. main().catch already worked around this with a 200ms setTimeout — extract exitAfterPinoFlush() and apply it to the process-level handlers too. 2. socket.send() in the relay transport adapter and ws.send() in sendToClient/sendBinaryToClient were unprotected. The ws library throws synchronously on send-after-close, and the readyState === 1 check has a race window. With an active foreground turn streaming agent_stream output, a peer-side disconnect could turn into an uncaughtException — which (per #1) then exited without logging. Wrap the three send sites in try/catch and warn-log on failure; let onclose/onerror drive cleanup. The readyState fast-path stays as an optimization. Refs getpaseo/paseo#612 Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
76c5ec56b5
commit
c1cfc7c743
@@ -207,19 +207,23 @@ async function main() {
|
|||||||
|
|
||||||
process.on("uncaughtException", (err) => {
|
process.on("uncaughtException", (err) => {
|
||||||
logger.fatal({ err }, "Uncaught exception — daemon crashing");
|
logger.fatal({ err }, "Uncaught exception — daemon crashing");
|
||||||
process.exit(1);
|
exitAfterPinoFlush();
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on("unhandledRejection", (reason) => {
|
process.on("unhandledRejection", (reason) => {
|
||||||
logger.fatal({ err: reason }, "Unhandled promise rejection — daemon crashing");
|
logger.fatal({ err: reason }, "Unhandled promise rejection — daemon crashing");
|
||||||
process.exit(1);
|
exitAfterPinoFlush();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Give pino async streams a moment to flush the fatal log entry to daemon.log
|
||||||
|
// before the process exits. Without this, the last few entries that explain
|
||||||
|
// why the daemon crashed can be lost.
|
||||||
|
function exitAfterPinoFlush(): void {
|
||||||
|
setTimeout(() => process.exit(1), 200);
|
||||||
|
}
|
||||||
|
|
||||||
main().catch((err) => {
|
main().catch((err) => {
|
||||||
process.stderr.write(`${err instanceof Error ? (err.stack ?? err.message) : String(err)}\n`);
|
process.stderr.write(`${err instanceof Error ? (err.stack ?? err.message) : String(err)}\n`);
|
||||||
// Give pino streams a moment to flush the fatal log entry to daemon.log
|
exitAfterPinoFlush();
|
||||||
// before the process exits. Without this, async file streams may lose the
|
|
||||||
// last few entries that explain why the daemon crashed.
|
|
||||||
setTimeout(() => process.exit(1), 200);
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -396,7 +396,7 @@ async function attachEncryptedSocket(
|
|||||||
metadata?: ExternalSocketMetadata,
|
metadata?: ExternalSocketMetadata,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const relayTransport = createRelayTransportAdapter(socket);
|
const relayTransport = createRelayTransportAdapter(socket, logger);
|
||||||
const emitter = new EventEmitter();
|
const emitter = new EventEmitter();
|
||||||
const channel = await createDaemonChannel(relayTransport, daemonKeyPair, {
|
const channel = await createDaemonChannel(relayTransport, daemonKeyPair, {
|
||||||
onmessage: (data) => emitter.emit("message", data),
|
onmessage: (data) => emitter.emit("message", data),
|
||||||
@@ -418,9 +418,18 @@ async function attachEncryptedSocket(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function createRelayTransportAdapter(socket: WebSocket): RelayTransport {
|
function createRelayTransportAdapter(socket: WebSocket, logger: pino.Logger): RelayTransport {
|
||||||
const relayTransport: RelayTransport = {
|
const relayTransport: RelayTransport = {
|
||||||
send: (data) => socket.send(data),
|
send: (data) => {
|
||||||
|
try {
|
||||||
|
socket.send(data);
|
||||||
|
} catch (err) {
|
||||||
|
// Socket likely transitioned to closed between checks; let onclose/onerror
|
||||||
|
// drive cleanup. Without this guard the synchronous throw would propagate
|
||||||
|
// up as an uncaughtException and take down the daemon.
|
||||||
|
logger.warn({ err }, "relay_socket_send_failed");
|
||||||
|
}
|
||||||
|
},
|
||||||
close: (code?: number, reason?: string) => socket.close(code, reason),
|
close: (code?: number, reason?: string) => socket.close(code, reason),
|
||||||
onmessage: null,
|
onmessage: null,
|
||||||
onclose: null,
|
onclose: null,
|
||||||
|
|||||||
@@ -738,10 +738,17 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private sendToClient(ws: WebSocketLike, message: WSOutboundMessage): void {
|
private sendToClient(ws: WebSocketLike, message: WSOutboundMessage): void {
|
||||||
// WebSocket.OPEN = 1
|
// WebSocket.OPEN = 1. The check is a fast path; the socket can still
|
||||||
if (ws.readyState === 1) {
|
// transition to closed between here and ws.send(), so guard the send too —
|
||||||
|
// a synchronous throw here would propagate as an uncaughtException.
|
||||||
|
if (ws.readyState !== 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
ws.send(JSON.stringify(message));
|
ws.send(JSON.stringify(message));
|
||||||
this.recordOutboundMessage(message, ws);
|
this.recordOutboundMessage(message, ws);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn({ err }, "ws_send_failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -749,8 +756,12 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
if (ws.readyState !== 1) {
|
if (ws.readyState !== 1) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
ws.send(frame);
|
try {
|
||||||
this.recordOutboundBinaryFrame(ws);
|
ws.send(frame);
|
||||||
|
this.recordOutboundBinaryFrame(ws);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn({ err }, "ws_send_binary_failed");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private sendToConnection(connection: SessionConnection, message: WSOutboundMessage): void {
|
private sendToConnection(connection: SessionConnection, message: WSOutboundMessage): void {
|
||||||
|
|||||||
Reference in New Issue
Block a user