feat(dictate): support configurable ASR backends
This commit is contained in:
@@ -0,0 +1,507 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from pi_dictate_service.backend import PcmQueue, ProviderError, provider_event
|
||||
from pi_dictate_service.backends.meta import (
|
||||
META_ENDPOINT,
|
||||
META_MAX_BACKLOG_BYTES,
|
||||
META_SOCKET_IO_TIMEOUT,
|
||||
MetaBackend,
|
||||
_default_transport as default_meta_transport,
|
||||
)
|
||||
from pi_dictate_service.backends.openai import (
|
||||
OPENAI_ENDPOINT,
|
||||
OPENAI_SOCKET_IO_TIMEOUT,
|
||||
OpenAIBackend,
|
||||
_default_transport as default_openai_transport,
|
||||
)
|
||||
from pi_dictate_service.config import MetaConfig, OpenAIConfig
|
||||
|
||||
|
||||
FIXTURE_ROOT = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def fixture_events(provider: str) -> list[str]:
|
||||
return (FIXTURE_ROOT / provider / "events.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
|
||||
|
||||
def _capture_start_error(backend: object, errors: list[BaseException]) -> None:
|
||||
try:
|
||||
backend.start_turn()
|
||||
except BaseException as error:
|
||||
errors.append(error)
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self, events: list[str | bytes | None]) -> None:
|
||||
self.events = list(events)
|
||||
self.sent: list[str | bytes] = []
|
||||
self.closed = False
|
||||
|
||||
def send(self, value: str | bytes) -> None:
|
||||
self.sent.append(value)
|
||||
|
||||
def recv(self, timeout: float | None = None) -> str | bytes | None:
|
||||
if not self.events:
|
||||
return None
|
||||
return self.events.pop(0)
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class BlockingCloseTransport(FakeTransport):
|
||||
def __init__(self, events: list[str | bytes | None]) -> None:
|
||||
super().__init__(events)
|
||||
self.close_started = threading.Event()
|
||||
self.allow_close = threading.Event()
|
||||
|
||||
def close(self) -> None:
|
||||
self.close_started.set()
|
||||
self.allow_close.wait(1)
|
||||
super().close()
|
||||
|
||||
|
||||
class HttpFailure(RuntimeError):
|
||||
def __init__(self, status_code: int) -> None:
|
||||
super().__init__(f"HTTP {status_code}")
|
||||
self.response = SimpleNamespace(status_code=status_code)
|
||||
|
||||
|
||||
class BackendTests(unittest.TestCase):
|
||||
def test_production_transports_apply_a_socket_io_deadline(self) -> None:
|
||||
for factory, timeout, arguments in (
|
||||
(default_meta_transport, META_SOCKET_IO_TIMEOUT, (META_ENDPOINT,)),
|
||||
(
|
||||
default_openai_transport,
|
||||
OPENAI_SOCKET_IO_TIMEOUT,
|
||||
(OPENAI_ENDPOINT,),
|
||||
),
|
||||
):
|
||||
with self.subTest(factory=factory.__module__):
|
||||
transport = SimpleNamespace(socket=Mock())
|
||||
with patch("websockets.sync.client.connect", return_value=transport):
|
||||
if factory is default_openai_transport:
|
||||
factory(*arguments, headers={"Authorization": "Bearer test"})
|
||||
else:
|
||||
factory(*arguments)
|
||||
transport.socket.settimeout.assert_called_once_with(timeout)
|
||||
|
||||
def make_files(self) -> tuple[Path, Path, tempfile.TemporaryDirectory[str]]:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
root = Path(temporary.name)
|
||||
key = root / "key"
|
||||
key.write_text("secret", encoding="utf-8")
|
||||
key.chmod(0o600)
|
||||
terms = root / "terms"
|
||||
terms.write_text("JSON\n", encoding="utf-8")
|
||||
return key, terms, temporary
|
||||
|
||||
def test_meta_waits_for_ack_sends_documented_handshake_and_finalizes_after_tail(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
acknowledgement = (FIXTURE_ROOT / "meta" / "session_ack.json").read_text(encoding="utf-8").strip()
|
||||
transport = FakeTransport([acknowledgement, *fixture_events("meta"), None])
|
||||
connections: list[str] = []
|
||||
|
||||
def connect(endpoint: str) -> FakeTransport:
|
||||
connections.append(endpoint)
|
||||
return transport
|
||||
|
||||
backend = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms, language_bias=("en",)),
|
||||
transport_factory=connect,
|
||||
timeout=1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
turn.enqueue_pcm(b"pcm")
|
||||
self.assertEqual(turn.finish(), "final fixture transcript")
|
||||
|
||||
self.assertEqual(connections, [META_ENDPOINT])
|
||||
handshake = json.loads(transport.sent[0])
|
||||
self.assertEqual(handshake["authorization"]["accessToken"], "Bearer secret")
|
||||
self.assertEqual(handshake["audioEncoding"], "PCM_24KHZ")
|
||||
self.assertEqual(handshake["mode"], "PUSH_TO_TALK")
|
||||
self.assertFalse(handshake["emitAudioProgress"])
|
||||
self.assertEqual(handshake["languageBias"], ["en"])
|
||||
self.assertEqual(handshake["keywords"], ["JSON"])
|
||||
self.assertEqual(transport.sent[1], b"pcm")
|
||||
self.assertEqual(json.loads(transport.sent[2]), {"type": "endStream"})
|
||||
|
||||
def test_openai_uses_ga_session_shape_and_correlates_committed_item(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = FakeTransport(fixture_events("openai"))
|
||||
connections: list[tuple[str, dict[str, str]]] = []
|
||||
|
||||
def connect(endpoint: str, *, headers: dict[str, str]) -> FakeTransport:
|
||||
connections.append((endpoint, headers))
|
||||
return transport
|
||||
|
||||
backend = OpenAIBackend(
|
||||
OpenAIConfig(
|
||||
"gpt-live-transcribe",
|
||||
key,
|
||||
terms,
|
||||
delay="low",
|
||||
prompt="Coding",
|
||||
languages=("en",),
|
||||
),
|
||||
transport_factory=connect,
|
||||
timeout=1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
turn.enqueue_pcm(b"pcm")
|
||||
self.assertEqual(turn.finish(), "final fixture transcript")
|
||||
|
||||
self.assertEqual(
|
||||
connections,
|
||||
[(OPENAI_ENDPOINT, {"Authorization": "Bearer secret"})],
|
||||
)
|
||||
update = json.loads(transport.sent[0])
|
||||
session = update["session"]
|
||||
self.assertEqual(session["type"], "transcription")
|
||||
input_audio = session["audio"]["input"]
|
||||
self.assertEqual(input_audio["format"], {"type": "audio/pcm", "rate": 24000})
|
||||
self.assertEqual(input_audio["transcription"]["languages"], ["en"])
|
||||
self.assertEqual(input_audio["transcription"]["delay"], "low")
|
||||
self.assertEqual(input_audio["transcription"]["keywords"], ["JSON"])
|
||||
self.assertIsNone(input_audio["turn_detection"])
|
||||
self.assertEqual(json.loads(transport.sent[-1]), {"type": "input_audio_buffer.commit"})
|
||||
self.assertFalse(any(isinstance(item, str) and "response.create" in item for item in transport.sent))
|
||||
|
||||
def test_meta_rejects_malformed_ack_and_auth_error(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
malformed = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: FakeTransport(['{"sessionId":"s","unexpected":1}']),
|
||||
timeout=0.1,
|
||||
)
|
||||
with self.assertRaisesRegex(ProviderError, "acknowledgement"):
|
||||
malformed.start_turn()
|
||||
|
||||
auth = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: FakeTransport(['{"type":"error","code":"401"}']),
|
||||
timeout=0.1,
|
||||
)
|
||||
with self.assertRaisesRegex(ProviderError, "Meta transcription service reported") as error:
|
||||
auth.start_turn()
|
||||
self.assertEqual(error.exception.code, "meta_auth")
|
||||
with self.assertRaises(ValueError):
|
||||
MetaBackend(MetaConfig("muse-voice-transcribe-1.0", key, terms), max_queue_bytes=META_MAX_BACKLOG_BYTES + 1)
|
||||
|
||||
def test_cloud_turn_rejects_partial_only_close_and_queue_overflow(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = FakeTransport(['{"sessionId":"s"}', '{"type":"transcript","final":false,"text":"partial"}', None])
|
||||
backend = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: transport,
|
||||
timeout=0.1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
with self.assertRaisesRegex(ProviderError, "final transcript"):
|
||||
turn.finish()
|
||||
|
||||
queue = PcmQueue(max_bytes=2, max_chunks=1)
|
||||
queue.put(b"12")
|
||||
with self.assertRaisesRegex(ProviderError, "full"):
|
||||
queue.put(b"3")
|
||||
|
||||
def test_cancel_before_prepare_worker_runs_prevents_connection(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
for backend in (
|
||||
MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda *_args, **_kwargs: self.fail("Meta connected"),
|
||||
),
|
||||
OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda *_args, **_kwargs: self.fail("OpenAI connected"),
|
||||
),
|
||||
):
|
||||
with self.subTest(backend=backend.descriptor.backend):
|
||||
backend.begin_prepare()
|
||||
backend.cancel_prepare()
|
||||
with self.assertRaises(ProviderError) as error:
|
||||
backend.start_turn()
|
||||
self.assertEqual(error.exception.code, "cancelled")
|
||||
|
||||
def test_cancel_during_connect_prevents_authenticated_handshake(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
for backend_name in ("meta", "openai"):
|
||||
with self.subTest(backend=backend_name):
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
transport = FakeTransport([])
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def connect(*_args: object, **_kwargs: object) -> FakeTransport:
|
||||
entered.set()
|
||||
release.wait(1)
|
||||
return transport
|
||||
|
||||
if backend_name == "meta":
|
||||
backend = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=connect,
|
||||
timeout=0.1,
|
||||
)
|
||||
else:
|
||||
backend = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=connect,
|
||||
timeout=0.1,
|
||||
)
|
||||
worker = threading.Thread(
|
||||
target=lambda: _capture_start_error(backend, errors), daemon=True
|
||||
)
|
||||
worker.start()
|
||||
self.assertTrue(entered.wait(1))
|
||||
backend.cancel_prepare()
|
||||
release.set()
|
||||
worker.join(1)
|
||||
|
||||
self.assertFalse(worker.is_alive())
|
||||
self.assertEqual(getattr(errors[0], "code", None), "cancelled")
|
||||
self.assertEqual(transport.sent, [])
|
||||
|
||||
def test_meta_discards_high_volume_partials_instead_of_overflowing_event_queue(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
partial = '{"type":"transcript","final":false,"text":"partial"}'
|
||||
transport = FakeTransport([
|
||||
'{"sessionId":"s"}',
|
||||
*([partial] * 200),
|
||||
'{"type":"transcript","final":true,"text":"final"}',
|
||||
None,
|
||||
])
|
||||
turn = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: transport,
|
||||
timeout=1,
|
||||
).start_turn()
|
||||
|
||||
self.assertEqual(turn.finish(), "final")
|
||||
|
||||
def test_cancel_never_sends_a_provider_end_marker(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transports_and_turns = []
|
||||
meta_transport = FakeTransport(['{"sessionId":"s"}'])
|
||||
transports_and_turns.append((
|
||||
meta_transport,
|
||||
MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: meta_transport,
|
||||
timeout=1,
|
||||
).start_turn(),
|
||||
))
|
||||
openai_transport = FakeTransport([
|
||||
'{"type":"session.created"}', '{"type":"session.updated"}'
|
||||
])
|
||||
transports_and_turns.append((
|
||||
openai_transport,
|
||||
OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: openai_transport,
|
||||
timeout=1,
|
||||
).start_turn(),
|
||||
))
|
||||
|
||||
for transport, turn in transports_and_turns:
|
||||
turn.cancel()
|
||||
deadline = time.monotonic() + 1
|
||||
while not turn.cleanup_complete and time.monotonic() < deadline:
|
||||
turn.close()
|
||||
sent_types = [
|
||||
value.get("type")
|
||||
for raw in transport.sent
|
||||
if isinstance(raw, str) and isinstance((value := json.loads(raw)), dict)
|
||||
]
|
||||
self.assertNotIn("endStream", sent_types)
|
||||
self.assertNotIn("input_audio_buffer.commit", sent_types)
|
||||
|
||||
def test_cancel_does_not_block_on_a_slow_transport_close(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = BlockingCloseTransport(['{"sessionId":"s"}'])
|
||||
turn = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: transport,
|
||||
timeout=0.1,
|
||||
).start_turn()
|
||||
|
||||
started = time.monotonic()
|
||||
turn.cancel()
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
self.assertLess(elapsed, 0.05)
|
||||
self.assertTrue(transport.close_started.wait(1))
|
||||
self.assertFalse(turn.cleanup_complete)
|
||||
transport.allow_close.set()
|
||||
deadline = time.monotonic() + 1
|
||||
while not turn.cleanup_complete and time.monotonic() < deadline:
|
||||
turn.close()
|
||||
self.assertTrue(turn.cleanup_complete)
|
||||
|
||||
def test_http_failures_use_websockets_response_status(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
meta = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: (_ for _ in ()).throw(HttpFailure(401)),
|
||||
)
|
||||
with self.assertRaises(ProviderError) as meta_error:
|
||||
meta.start_turn()
|
||||
self.assertEqual(meta_error.exception.code, "meta_auth")
|
||||
|
||||
openai = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: (_ for _ in ()).throw(HttpFailure(429)),
|
||||
)
|
||||
with self.assertRaises(ProviderError) as openai_error:
|
||||
openai.start_turn()
|
||||
self.assertEqual(openai_error.exception.code, "openai_rate_limit")
|
||||
|
||||
def test_cloud_turn_reports_provider_close_during_recording(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
backend = MetaBackend(
|
||||
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
||||
transport_factory=lambda _endpoint: FakeTransport([
|
||||
'{"sessionId":"s"}',
|
||||
None,
|
||||
]),
|
||||
timeout=0.1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
deadline = time.monotonic() + 1
|
||||
failure = turn.poll_error()
|
||||
while failure is None and time.monotonic() < deadline:
|
||||
time.sleep(0.001)
|
||||
failure = turn.poll_error()
|
||||
|
||||
self.assertIsNotNone(failure)
|
||||
self.assertEqual(failure.code, "meta_premature_close")
|
||||
|
||||
def test_provider_event_rejects_recursive_values(self) -> None:
|
||||
recursive: dict[str, object] = {}
|
||||
recursive["self"] = recursive
|
||||
with self.assertRaises(ProviderError):
|
||||
provider_event(recursive)
|
||||
|
||||
def test_openai_rejects_malformed_error_and_unmatched_completion(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
malformed = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: FakeTransport([
|
||||
'{"type":"session.created"}',
|
||||
'{"type":"error","error":{"message":"bad"}}',
|
||||
]),
|
||||
timeout=0.1,
|
||||
)
|
||||
with self.assertRaisesRegex(ProviderError, "error event"):
|
||||
malformed.start_turn()
|
||||
|
||||
transport = FakeTransport([
|
||||
'{"type":"session.created"}',
|
||||
'{"type":"session.updated"}',
|
||||
'{"type":"input_audio_buffer.committed","item_id":"new"}',
|
||||
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"old","transcript":"stale"}',
|
||||
None,
|
||||
])
|
||||
backend = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: transport,
|
||||
timeout=0.1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
with self.assertRaises(ProviderError) as error:
|
||||
turn.finish()
|
||||
self.assertIn(error.exception.code, {"openai_premature_close", "openai_timeout"})
|
||||
|
||||
def test_openai_commit_oriented_alternative_uses_explicit_commit(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = FakeTransport(fixture_events("openai"))
|
||||
turn = OpenAIBackend(
|
||||
OpenAIConfig("gpt-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: transport,
|
||||
timeout=1,
|
||||
).start_turn()
|
||||
turn.enqueue_pcm(b"pcm")
|
||||
|
||||
self.assertEqual(turn.finish(), "final fixture transcript")
|
||||
update = json.loads(transport.sent[0])
|
||||
self.assertEqual(
|
||||
update["session"]["audio"]["input"]["transcription"]["model"],
|
||||
"gpt-transcribe",
|
||||
)
|
||||
self.assertEqual(json.loads(transport.sent[-1]), {"type": "input_audio_buffer.commit"})
|
||||
|
||||
def test_openai_rejects_unknown_delay_from_direct_callers(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
with self.assertRaisesRegex(ValueError, "delay"):
|
||||
OpenAIBackend(OpenAIConfig("gpt-live-transcribe", key, terms, delay="fast"))
|
||||
|
||||
def test_openai_receiver_failure_overrides_an_already_queued_completion(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = FakeTransport([
|
||||
'{"type":"session.created"}',
|
||||
'{"type":"session.updated"}',
|
||||
'{"type":"input_audio_buffer.committed","item_id":"i"}',
|
||||
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"i","transcript":"unsafe"}',
|
||||
'{}',
|
||||
])
|
||||
turn = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: transport,
|
||||
timeout=1,
|
||||
).start_turn()
|
||||
deadline = time.monotonic() + 1
|
||||
while turn._receiver_error is None and time.monotonic() < deadline:
|
||||
time.sleep(0.001)
|
||||
|
||||
with self.assertRaises(ProviderError) as error:
|
||||
turn.finish()
|
||||
self.assertEqual(error.exception.code, "openai_malformed_event")
|
||||
|
||||
def test_openai_audio_is_bounded_base64(self) -> None:
|
||||
key, terms, temporary = self.make_files()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
transport = FakeTransport(
|
||||
['{"type":"session.created"}', '{"type":"session.updated"}',
|
||||
'{"type":"input_audio_buffer.committed","item_id":"i"}',
|
||||
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"i","transcript":""}']
|
||||
)
|
||||
backend = OpenAIBackend(
|
||||
OpenAIConfig("gpt-live-transcribe", key, terms),
|
||||
transport_factory=lambda _endpoint, **_kwargs: transport,
|
||||
timeout=1,
|
||||
)
|
||||
turn = backend.start_turn()
|
||||
turn.enqueue_pcm(b"\x01\x02")
|
||||
self.assertEqual(turn.finish(), "")
|
||||
append = json.loads(transport.sent[1])
|
||||
self.assertEqual(base64.b64decode(append["audio"]), b"\x01\x02")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user