feat(dictate): add voice-reactive listening waveform

services/dictate/src/pi_dictate_service/model.py

@@ -86,12 +86,13 @@ class NemoParakeetModel:
         os.environ["NEMO_OFFLINE"] = "1"
 
         import numpy
-        from nemo.collections.asr.models import ASRModel
+        import torch
+        from nemo.collections.asr.models import EncDecRNNTBPEModel
         from omegaconf import open_dict
 
-        model = ASRModel.restore_from(
+        model = EncDecRNNTBPEModel.restore_from(
             restore_path=str(self._model_path),
-            map_location="cuda",
+            map_location=torch.device("cuda"),
         )
         model.to("cuda")
         model.eval()

services/dictate/src/pi_dictate_service/protocol.py

@@ -1,4 +1,4 @@
-"""Bounded newline-delimited JSON protocol primitives for dictate protocol v1."""
+"""Bounded newline-delimited JSON protocol primitives for dictate protocol v2."""
 
 from __future__ import annotations
 
@@ -6,7 +6,7 @@ from dataclasses import dataclass
 import json
 from typing import Any, Mapping
 
-PROTOCOL_VERSION = 1
+PROTOCOL_VERSION = 2
 MAX_FRAME_BYTES = 64 * 1024
 MAX_TRANSCRIPT_BYTES = 32 * 1024
 MAX_ERROR_DETAIL_BYTES = 2 * 1024

services/dictate/src/pi_dictate_service/recorder.py

@@ -2,10 +2,13 @@
 
 from __future__ import annotations
 
+from array import array
 from dataclasses import dataclass
+import math
 import os
 import signal
 import subprocess
+import sys
 import threading
 import time
 from typing import BinaryIO, Callable, Protocol
@@ -24,6 +27,7 @@ PW_RECORD_COMMAND = [
     "-",
 ]
 MAX_STDERR_BYTES = 64 * 1024
+AUDIO_LEVEL_WINDOW_BYTES = 3_200  # 100 ms of mono 16 kHz signed-16 PCM
 
 
 class Process(Protocol):
@@ -95,12 +99,18 @@ class PwRecorder:
         self._duration_limited = False
         self._duration_stop_at: float | None = None
         self._reported = False
+        self._audio_level = 0.0
 
     @property
     def buffered_bytes(self) -> int:
         with self._lock:
             return len(self._pcm)
 
+    @property
+    def audio_level(self) -> float:
+        with self._lock:
+            return self._audio_level
+
     def start(self) -> None:
         if self._process is not None:
             raise RecorderError("invalid_state", "recorder is already started")
@@ -223,10 +233,15 @@ class PwRecorder:
 
     def _drain_stdout(self) -> None:
         process = self._require_process()
+        level_window = bytearray()
         while True:
-            chunk = process.stdout.read(64 * 1024)
+            chunk = process.stdout.read(AUDIO_LEVEL_WINDOW_BYTES)
             if not chunk:
                 return
+            level_window.extend(chunk)
+            while len(level_window) >= AUDIO_LEVEL_WINDOW_BYTES:
+                self._update_audio_level(level_window[:AUDIO_LEVEL_WINDOW_BYTES])
+                del level_window[:AUDIO_LEVEL_WINDOW_BYTES]
             with self._lock:
                 if self._duration_limited:
                     continue
@@ -240,6 +255,19 @@ class PwRecorder:
             if process.poll() is None:
                 self._signal_group(process, signal.SIGTERM)
 
+    def _update_audio_level(self, chunk: bytes) -> None:
+        sample_bytes = len(chunk) - (len(chunk) % 2)
+        if sample_bytes == 0:
+            return
+        samples = array("h")
+        samples.frombytes(chunk[:sample_bytes])
+        if sys.byteorder != "little":
+            samples.byteswap()
+        rms = math.sqrt(sum(sample * sample for sample in samples) / len(samples)) / 32768.0
+        level = 0.0 if rms == 0 else max(0.0, min(1.0, (20 * math.log10(rms) + 60) / 60))
+        with self._lock:
+            self._audio_level = max(level, self._audio_level * 0.65)
+
     def _drain_stderr(self) -> None:
         process = self._require_process()
         while True:

services/dictate/src/pi_dictate_service/server.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from collections.abc import Sequence
 from concurrent.futures import Executor, Future, ThreadPoolExecutor
 from dataclasses import dataclass
+import math
 import os
 from queue import Empty, SimpleQueue
 import selectors
@@ -30,9 +31,13 @@ from .recorder import Capture, PwRecorder, RecorderError
 DEFAULT_IDLE_SECONDS = 15 * 60
 MAX_CLIENT_CONNECTIONS = 16
 MAX_CONNECTION_OUTPUT_BYTES = MAX_FRAME_BYTES
+LEVEL_EVENT_INTERVAL_SECONDS = 0.1
 
 
 class Recorder(Protocol):
+    @property
+    def audio_level(self) -> float: ...
+
     def start(self) -> None: ...
     def stop(self) -> Capture: ...
     def cancel(self) -> None: ...
@@ -53,6 +58,7 @@ class _Generation:
     state: str
     recorder: Recorder | None = None
     tombstoned: bool = False
+    last_level_event_at: float | None = None
 
 
 @dataclass(frozen=True)
@@ -156,6 +162,8 @@ class DictateServiceCore:
             if failure is not None:
                 self._async_error(active, failure.code, failure.detail)
                 self._finish_generation(active, notify_idle=True)
+                return
+            self._emit_audio_level(active)
 
     def drain_events(self) -> list[Outbound]:
         events, self._events = self._events, []
@@ -216,8 +224,31 @@ class DictateServiceCore:
             self._finish_generation(active, notify_idle=True)
             return
         active.recorder = recorder
+        active.last_level_event_at = self._clock()
         self._state(active, "listening")
 
+    def _emit_audio_level(self, active: _Generation) -> None:
+        now = self._clock()
+        if (
+            active.last_level_event_at is not None
+            and now - active.last_level_event_at < LEVEL_EVENT_INTERVAL_SECONDS
+        ):
+            return
+        level = active.recorder.audio_level if active.recorder is not None else 0.0
+        if not math.isfinite(level):
+            level = 0.0
+        active.last_level_event_at = now
+        self._emit(
+            active.owner,
+            {
+                "version": PROTOCOL_VERSION,
+                "type": "level",
+                "requestId": active.request_id,
+                "generationId": active.generation_id,
+                "level": round(max(0.0, min(1.0, level)), 3),
+            },
+        )
+
     def _stop(self, client_id: str, request: Request) -> None:
         active = self._owned_generation(client_id, request)
         if active is None:
@@ -433,7 +464,7 @@ def inherited_listener(environ: dict[str, str] | None = None) -> socket.socket:
 
 
 def serve(listener: socket.socket, core: DictateServiceCore) -> None:
-    """Serve protocol v1 on an explicit listener until the core becomes idle."""
+    """Serve protocol v2 on an explicit listener until the core becomes idle."""
     selector = selectors.DefaultSelector()
     connections: dict[str, _Connection] = {}
     sequence = 0