508 lines
20 KiB
Python
508 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
from pi_dictate_service.backend import PcmQueue, ProviderError, provider_event
|
|
from pi_dictate_service.backends.meta import (
|
|
META_ENDPOINT,
|
|
META_MAX_BACKLOG_BYTES,
|
|
META_SOCKET_IO_TIMEOUT,
|
|
MetaBackend,
|
|
_default_transport as default_meta_transport,
|
|
)
|
|
from pi_dictate_service.backends.openai import (
|
|
OPENAI_ENDPOINT,
|
|
OPENAI_SOCKET_IO_TIMEOUT,
|
|
OpenAIBackend,
|
|
_default_transport as default_openai_transport,
|
|
)
|
|
from pi_dictate_service.config import MetaConfig, OpenAIConfig
|
|
|
|
|
|
FIXTURE_ROOT = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def fixture_events(provider: str) -> list[str]:
|
|
return (FIXTURE_ROOT / provider / "events.jsonl").read_text(encoding="utf-8").splitlines()
|
|
|
|
|
|
def _capture_start_error(backend: object, errors: list[BaseException]) -> None:
|
|
try:
|
|
backend.start_turn()
|
|
except BaseException as error:
|
|
errors.append(error)
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self, events: list[str | bytes | None]) -> None:
|
|
self.events = list(events)
|
|
self.sent: list[str | bytes] = []
|
|
self.closed = False
|
|
|
|
def send(self, value: str | bytes) -> None:
|
|
self.sent.append(value)
|
|
|
|
def recv(self, timeout: float | None = None) -> str | bytes | None:
|
|
if not self.events:
|
|
return None
|
|
return self.events.pop(0)
|
|
|
|
def close(self) -> None:
|
|
self.closed = True
|
|
|
|
|
|
class BlockingCloseTransport(FakeTransport):
|
|
def __init__(self, events: list[str | bytes | None]) -> None:
|
|
super().__init__(events)
|
|
self.close_started = threading.Event()
|
|
self.allow_close = threading.Event()
|
|
|
|
def close(self) -> None:
|
|
self.close_started.set()
|
|
self.allow_close.wait(1)
|
|
super().close()
|
|
|
|
|
|
class HttpFailure(RuntimeError):
|
|
def __init__(self, status_code: int) -> None:
|
|
super().__init__(f"HTTP {status_code}")
|
|
self.response = SimpleNamespace(status_code=status_code)
|
|
|
|
|
|
class BackendTests(unittest.TestCase):
|
|
def test_production_transports_apply_a_socket_io_deadline(self) -> None:
|
|
for factory, timeout, arguments in (
|
|
(default_meta_transport, META_SOCKET_IO_TIMEOUT, (META_ENDPOINT,)),
|
|
(
|
|
default_openai_transport,
|
|
OPENAI_SOCKET_IO_TIMEOUT,
|
|
(OPENAI_ENDPOINT,),
|
|
),
|
|
):
|
|
with self.subTest(factory=factory.__module__):
|
|
transport = SimpleNamespace(socket=Mock())
|
|
with patch("websockets.sync.client.connect", return_value=transport):
|
|
if factory is default_openai_transport:
|
|
factory(*arguments, headers={"Authorization": "Bearer test"})
|
|
else:
|
|
factory(*arguments)
|
|
transport.socket.settimeout.assert_called_once_with(timeout)
|
|
|
|
def make_files(self) -> tuple[Path, Path, tempfile.TemporaryDirectory[str]]:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
root = Path(temporary.name)
|
|
key = root / "key"
|
|
key.write_text("secret", encoding="utf-8")
|
|
key.chmod(0o600)
|
|
terms = root / "terms"
|
|
terms.write_text("JSON\n", encoding="utf-8")
|
|
return key, terms, temporary
|
|
|
|
def test_meta_waits_for_ack_sends_documented_handshake_and_finalizes_after_tail(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
acknowledgement = (FIXTURE_ROOT / "meta" / "session_ack.json").read_text(encoding="utf-8").strip()
|
|
transport = FakeTransport([acknowledgement, *fixture_events("meta"), None])
|
|
connections: list[str] = []
|
|
|
|
def connect(endpoint: str) -> FakeTransport:
|
|
connections.append(endpoint)
|
|
return transport
|
|
|
|
backend = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms, language_bias=("en",)),
|
|
transport_factory=connect,
|
|
timeout=1,
|
|
)
|
|
turn = backend.start_turn()
|
|
turn.enqueue_pcm(b"pcm")
|
|
self.assertEqual(turn.finish(), "final fixture transcript")
|
|
|
|
self.assertEqual(connections, [META_ENDPOINT])
|
|
handshake = json.loads(transport.sent[0])
|
|
self.assertEqual(handshake["authorization"]["accessToken"], "Bearer secret")
|
|
self.assertEqual(handshake["audioEncoding"], "PCM_24KHZ")
|
|
self.assertEqual(handshake["mode"], "PUSH_TO_TALK")
|
|
self.assertFalse(handshake["emitAudioProgress"])
|
|
self.assertEqual(handshake["languageBias"], ["en"])
|
|
self.assertEqual(handshake["keywords"], ["JSON"])
|
|
self.assertEqual(transport.sent[1], b"pcm")
|
|
self.assertEqual(json.loads(transport.sent[2]), {"type": "endStream"})
|
|
|
|
def test_openai_uses_ga_session_shape_and_correlates_committed_item(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = FakeTransport(fixture_events("openai"))
|
|
connections: list[tuple[str, dict[str, str]]] = []
|
|
|
|
def connect(endpoint: str, *, headers: dict[str, str]) -> FakeTransport:
|
|
connections.append((endpoint, headers))
|
|
return transport
|
|
|
|
backend = OpenAIBackend(
|
|
OpenAIConfig(
|
|
"gpt-live-transcribe",
|
|
key,
|
|
terms,
|
|
delay="low",
|
|
prompt="Coding",
|
|
languages=("en",),
|
|
),
|
|
transport_factory=connect,
|
|
timeout=1,
|
|
)
|
|
turn = backend.start_turn()
|
|
turn.enqueue_pcm(b"pcm")
|
|
self.assertEqual(turn.finish(), "final fixture transcript")
|
|
|
|
self.assertEqual(
|
|
connections,
|
|
[(OPENAI_ENDPOINT, {"Authorization": "Bearer secret"})],
|
|
)
|
|
update = json.loads(transport.sent[0])
|
|
session = update["session"]
|
|
self.assertEqual(session["type"], "transcription")
|
|
input_audio = session["audio"]["input"]
|
|
self.assertEqual(input_audio["format"], {"type": "audio/pcm", "rate": 24000})
|
|
self.assertEqual(input_audio["transcription"]["languages"], ["en"])
|
|
self.assertEqual(input_audio["transcription"]["delay"], "low")
|
|
self.assertEqual(input_audio["transcription"]["keywords"], ["JSON"])
|
|
self.assertIsNone(input_audio["turn_detection"])
|
|
self.assertEqual(json.loads(transport.sent[-1]), {"type": "input_audio_buffer.commit"})
|
|
self.assertFalse(any(isinstance(item, str) and "response.create" in item for item in transport.sent))
|
|
|
|
def test_meta_rejects_malformed_ack_and_auth_error(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
malformed = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: FakeTransport(['{"sessionId":"s","unexpected":1}']),
|
|
timeout=0.1,
|
|
)
|
|
with self.assertRaisesRegex(ProviderError, "acknowledgement"):
|
|
malformed.start_turn()
|
|
|
|
auth = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: FakeTransport(['{"type":"error","code":"401"}']),
|
|
timeout=0.1,
|
|
)
|
|
with self.assertRaisesRegex(ProviderError, "Meta transcription service reported") as error:
|
|
auth.start_turn()
|
|
self.assertEqual(error.exception.code, "meta_auth")
|
|
with self.assertRaises(ValueError):
|
|
MetaBackend(MetaConfig("muse-voice-transcribe-1.0", key, terms), max_queue_bytes=META_MAX_BACKLOG_BYTES + 1)
|
|
|
|
def test_cloud_turn_rejects_partial_only_close_and_queue_overflow(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = FakeTransport(['{"sessionId":"s"}', '{"type":"transcript","final":false,"text":"partial"}', None])
|
|
backend = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: transport,
|
|
timeout=0.1,
|
|
)
|
|
turn = backend.start_turn()
|
|
with self.assertRaisesRegex(ProviderError, "final transcript"):
|
|
turn.finish()
|
|
|
|
queue = PcmQueue(max_bytes=2, max_chunks=1)
|
|
queue.put(b"12")
|
|
with self.assertRaisesRegex(ProviderError, "full"):
|
|
queue.put(b"3")
|
|
|
|
def test_cancel_before_prepare_worker_runs_prevents_connection(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
for backend in (
|
|
MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda *_args, **_kwargs: self.fail("Meta connected"),
|
|
),
|
|
OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda *_args, **_kwargs: self.fail("OpenAI connected"),
|
|
),
|
|
):
|
|
with self.subTest(backend=backend.descriptor.backend):
|
|
backend.begin_prepare()
|
|
backend.cancel_prepare()
|
|
with self.assertRaises(ProviderError) as error:
|
|
backend.start_turn()
|
|
self.assertEqual(error.exception.code, "cancelled")
|
|
|
|
def test_cancel_during_connect_prevents_authenticated_handshake(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
for backend_name in ("meta", "openai"):
|
|
with self.subTest(backend=backend_name):
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
transport = FakeTransport([])
|
|
errors: list[BaseException] = []
|
|
|
|
def connect(*_args: object, **_kwargs: object) -> FakeTransport:
|
|
entered.set()
|
|
release.wait(1)
|
|
return transport
|
|
|
|
if backend_name == "meta":
|
|
backend = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=connect,
|
|
timeout=0.1,
|
|
)
|
|
else:
|
|
backend = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=connect,
|
|
timeout=0.1,
|
|
)
|
|
worker = threading.Thread(
|
|
target=lambda: _capture_start_error(backend, errors), daemon=True
|
|
)
|
|
worker.start()
|
|
self.assertTrue(entered.wait(1))
|
|
backend.cancel_prepare()
|
|
release.set()
|
|
worker.join(1)
|
|
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(getattr(errors[0], "code", None), "cancelled")
|
|
self.assertEqual(transport.sent, [])
|
|
|
|
def test_meta_discards_high_volume_partials_instead_of_overflowing_event_queue(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
partial = '{"type":"transcript","final":false,"text":"partial"}'
|
|
transport = FakeTransport([
|
|
'{"sessionId":"s"}',
|
|
*([partial] * 200),
|
|
'{"type":"transcript","final":true,"text":"final"}',
|
|
None,
|
|
])
|
|
turn = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: transport,
|
|
timeout=1,
|
|
).start_turn()
|
|
|
|
self.assertEqual(turn.finish(), "final")
|
|
|
|
def test_cancel_never_sends_a_provider_end_marker(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transports_and_turns = []
|
|
meta_transport = FakeTransport(['{"sessionId":"s"}'])
|
|
transports_and_turns.append((
|
|
meta_transport,
|
|
MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: meta_transport,
|
|
timeout=1,
|
|
).start_turn(),
|
|
))
|
|
openai_transport = FakeTransport([
|
|
'{"type":"session.created"}', '{"type":"session.updated"}'
|
|
])
|
|
transports_and_turns.append((
|
|
openai_transport,
|
|
OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: openai_transport,
|
|
timeout=1,
|
|
).start_turn(),
|
|
))
|
|
|
|
for transport, turn in transports_and_turns:
|
|
turn.cancel()
|
|
deadline = time.monotonic() + 1
|
|
while not turn.cleanup_complete and time.monotonic() < deadline:
|
|
turn.close()
|
|
sent_types = [
|
|
value.get("type")
|
|
for raw in transport.sent
|
|
if isinstance(raw, str) and isinstance((value := json.loads(raw)), dict)
|
|
]
|
|
self.assertNotIn("endStream", sent_types)
|
|
self.assertNotIn("input_audio_buffer.commit", sent_types)
|
|
|
|
def test_cancel_does_not_block_on_a_slow_transport_close(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = BlockingCloseTransport(['{"sessionId":"s"}'])
|
|
turn = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: transport,
|
|
timeout=0.1,
|
|
).start_turn()
|
|
|
|
started = time.monotonic()
|
|
turn.cancel()
|
|
elapsed = time.monotonic() - started
|
|
|
|
self.assertLess(elapsed, 0.05)
|
|
self.assertTrue(transport.close_started.wait(1))
|
|
self.assertFalse(turn.cleanup_complete)
|
|
transport.allow_close.set()
|
|
deadline = time.monotonic() + 1
|
|
while not turn.cleanup_complete and time.monotonic() < deadline:
|
|
turn.close()
|
|
self.assertTrue(turn.cleanup_complete)
|
|
|
|
def test_http_failures_use_websockets_response_status(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
meta = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: (_ for _ in ()).throw(HttpFailure(401)),
|
|
)
|
|
with self.assertRaises(ProviderError) as meta_error:
|
|
meta.start_turn()
|
|
self.assertEqual(meta_error.exception.code, "meta_auth")
|
|
|
|
openai = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: (_ for _ in ()).throw(HttpFailure(429)),
|
|
)
|
|
with self.assertRaises(ProviderError) as openai_error:
|
|
openai.start_turn()
|
|
self.assertEqual(openai_error.exception.code, "openai_rate_limit")
|
|
|
|
def test_cloud_turn_reports_provider_close_during_recording(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
backend = MetaBackend(
|
|
MetaConfig("muse-voice-transcribe-1.0", key, terms),
|
|
transport_factory=lambda _endpoint: FakeTransport([
|
|
'{"sessionId":"s"}',
|
|
None,
|
|
]),
|
|
timeout=0.1,
|
|
)
|
|
turn = backend.start_turn()
|
|
deadline = time.monotonic() + 1
|
|
failure = turn.poll_error()
|
|
while failure is None and time.monotonic() < deadline:
|
|
time.sleep(0.001)
|
|
failure = turn.poll_error()
|
|
|
|
self.assertIsNotNone(failure)
|
|
self.assertEqual(failure.code, "meta_premature_close")
|
|
|
|
def test_provider_event_rejects_recursive_values(self) -> None:
|
|
recursive: dict[str, object] = {}
|
|
recursive["self"] = recursive
|
|
with self.assertRaises(ProviderError):
|
|
provider_event(recursive)
|
|
|
|
def test_openai_rejects_malformed_error_and_unmatched_completion(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
malformed = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: FakeTransport([
|
|
'{"type":"session.created"}',
|
|
'{"type":"error","error":{"message":"bad"}}',
|
|
]),
|
|
timeout=0.1,
|
|
)
|
|
with self.assertRaisesRegex(ProviderError, "error event"):
|
|
malformed.start_turn()
|
|
|
|
transport = FakeTransport([
|
|
'{"type":"session.created"}',
|
|
'{"type":"session.updated"}',
|
|
'{"type":"input_audio_buffer.committed","item_id":"new"}',
|
|
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"old","transcript":"stale"}',
|
|
None,
|
|
])
|
|
backend = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: transport,
|
|
timeout=0.1,
|
|
)
|
|
turn = backend.start_turn()
|
|
with self.assertRaises(ProviderError) as error:
|
|
turn.finish()
|
|
self.assertIn(error.exception.code, {"openai_premature_close", "openai_timeout"})
|
|
|
|
def test_openai_commit_oriented_alternative_uses_explicit_commit(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = FakeTransport(fixture_events("openai"))
|
|
turn = OpenAIBackend(
|
|
OpenAIConfig("gpt-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: transport,
|
|
timeout=1,
|
|
).start_turn()
|
|
turn.enqueue_pcm(b"pcm")
|
|
|
|
self.assertEqual(turn.finish(), "final fixture transcript")
|
|
update = json.loads(transport.sent[0])
|
|
self.assertEqual(
|
|
update["session"]["audio"]["input"]["transcription"]["model"],
|
|
"gpt-transcribe",
|
|
)
|
|
self.assertEqual(json.loads(transport.sent[-1]), {"type": "input_audio_buffer.commit"})
|
|
|
|
def test_openai_rejects_unknown_delay_from_direct_callers(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
with self.assertRaisesRegex(ValueError, "delay"):
|
|
OpenAIBackend(OpenAIConfig("gpt-live-transcribe", key, terms, delay="fast"))
|
|
|
|
def test_openai_receiver_failure_overrides_an_already_queued_completion(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = FakeTransport([
|
|
'{"type":"session.created"}',
|
|
'{"type":"session.updated"}',
|
|
'{"type":"input_audio_buffer.committed","item_id":"i"}',
|
|
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"i","transcript":"unsafe"}',
|
|
'{}',
|
|
])
|
|
turn = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: transport,
|
|
timeout=1,
|
|
).start_turn()
|
|
deadline = time.monotonic() + 1
|
|
while turn._receiver_error is None and time.monotonic() < deadline:
|
|
time.sleep(0.001)
|
|
|
|
with self.assertRaises(ProviderError) as error:
|
|
turn.finish()
|
|
self.assertEqual(error.exception.code, "openai_malformed_event")
|
|
|
|
def test_openai_audio_is_bounded_base64(self) -> None:
|
|
key, terms, temporary = self.make_files()
|
|
self.addCleanup(temporary.cleanup)
|
|
transport = FakeTransport(
|
|
['{"type":"session.created"}', '{"type":"session.updated"}',
|
|
'{"type":"input_audio_buffer.committed","item_id":"i"}',
|
|
'{"type":"conversation.item.input_audio_transcription.completed","item_id":"i","transcript":""}']
|
|
)
|
|
backend = OpenAIBackend(
|
|
OpenAIConfig("gpt-live-transcribe", key, terms),
|
|
transport_factory=lambda _endpoint, **_kwargs: transport,
|
|
timeout=1,
|
|
)
|
|
turn = backend.start_turn()
|
|
turn.enqueue_pcm(b"\x01\x02")
|
|
self.assertEqual(turn.finish(), "")
|
|
append = json.loads(transport.sent[1])
|
|
self.assertEqual(base64.b64decode(append["audio"]), b"\x01\x02")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|