Add game data capture vertical slice
This commit is contained in:
@@ -0,0 +1,545 @@
|
||||
"""Bounded lifecycle wiring for local capture intake and utility storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import stat
|
||||
import time
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from contextlib import AbstractContextManager, contextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event, RLock, Thread, current_thread, main_thread
|
||||
from typing import Protocol, cast
|
||||
|
||||
import uvicorn
|
||||
|
||||
from game_data_capture.config import AppConfig
|
||||
from game_data_capture.ingest.clipboard import ClipboardIntake, IntakeReceipt, WatchConfig
|
||||
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE
|
||||
from game_data_capture.paths import RuntimePaths, prepare_private_directories, prepare_private_file
|
||||
from game_data_capture.pipeline.jobs import JobLeases
|
||||
from game_data_capture.storage import (
|
||||
CaptureCorpus,
|
||||
CaptureRepository,
|
||||
ContentAddressedBlobStore,
|
||||
StorageDatabase,
|
||||
)
|
||||
from game_data_capture.web.app import (
|
||||
CaptureControlPort,
|
||||
CaptureSessionStatus,
|
||||
ControlOutcome,
|
||||
WebConfig,
|
||||
create_app,
|
||||
)
|
||||
from game_data_capture.web.security import new_bootstrap_token
|
||||
|
||||
_POLICY_VERSION = "capture-v1"
|
||||
_PROCESSING_VERSION = "capture-v1"
|
||||
_POLL_SECONDS = 0.1
|
||||
|
||||
|
||||
class RuntimeFailure(RuntimeError):
|
||||
"""A safe-to-display runtime failure that does not disclose local inputs."""
|
||||
|
||||
|
||||
class IntakePort(Protocol):
|
||||
"""The accepted media descriptor boundary consumed by this runtime."""
|
||||
|
||||
def reconcile(self) -> list[str]: ...
|
||||
|
||||
def arm(self) -> None: ...
|
||||
|
||||
def tick(self) -> None: ...
|
||||
|
||||
def take_helper_receipt(self) -> dict[str, object] | None: ...
|
||||
|
||||
def accept_helper_receipt(
|
||||
self,
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
occurrence_id: str,
|
||||
processing_version: str,
|
||||
) -> bool: ...
|
||||
|
||||
def take(self) -> IntakeReceipt | None: ...
|
||||
|
||||
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: ...
|
||||
|
||||
def acknowledge(self, receipt: IntakeReceipt) -> bool: ...
|
||||
|
||||
def shutdown(self) -> None: ...
|
||||
|
||||
|
||||
class InstanceLock:
|
||||
"""Descriptor-held ownership that safely reuses an unlocked stale lock file."""
|
||||
|
||||
def __init__(self, file_descriptor: int) -> None:
|
||||
self._file_descriptor: int | None = file_descriptor
|
||||
|
||||
@classmethod
|
||||
def acquire(cls, path: Path, timeout_seconds: int) -> InstanceLock:
|
||||
try:
|
||||
prepare_private_file(path)
|
||||
descriptor = os.open(path, os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
except OSError as error:
|
||||
raise RuntimeFailure("capture instance lock is unavailable") from error
|
||||
try:
|
||||
info = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600:
|
||||
raise RuntimeFailure("capture instance lock is unsafe")
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return cls(descriptor)
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeFailure("another capture instance is active") from None
|
||||
time.sleep(min(0.05, max(0.0, deadline - time.monotonic())))
|
||||
except Exception:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
|
||||
def release(self) -> None:
|
||||
if self._file_descriptor is None:
|
||||
return
|
||||
descriptor, self._file_descriptor = self._file_descriptor, None
|
||||
with suppress(OSError):
|
||||
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
||||
with suppress(OSError):
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Resources:
|
||||
repository: CaptureRepository
|
||||
corpus: CaptureCorpus
|
||||
lock: InstanceLock
|
||||
|
||||
def close(self) -> None:
|
||||
self.lock.release()
|
||||
|
||||
|
||||
class _CaptureControl(CaptureControlPort):
|
||||
"""Serialize one optional armed intake beneath a running local service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime: CaptureRuntime,
|
||||
repository: CaptureRepository,
|
||||
) -> None:
|
||||
self._runtime = runtime
|
||||
self._repository = repository
|
||||
self._lock = RLock()
|
||||
self._cancelled: Event | None = None
|
||||
self._intake: IntakePort | None = None
|
||||
self._worker: Thread | None = None
|
||||
self._state = "disarmed"
|
||||
|
||||
def status(self) -> CaptureSessionStatus:
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
intake = self._intake
|
||||
depth = self._queue_depth(intake)
|
||||
return CaptureSessionStatus(
|
||||
armed=self._state in {"armed", "stopping"},
|
||||
queue_depth=depth,
|
||||
queue_capacity=self._runtime.config.limits.max_queue_items,
|
||||
)
|
||||
|
||||
def start_session(self) -> ControlOutcome:
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
if self._state == "armed":
|
||||
return "already_armed"
|
||||
if self._state == "stopping":
|
||||
return "conflict"
|
||||
if self._worker is not None:
|
||||
return "conflict"
|
||||
intake: IntakePort | None = None
|
||||
try:
|
||||
intake = self._runtime.intake_factory(self._runtime._watch_config())
|
||||
# Durable media always becomes eligible before the watcher is armed.
|
||||
intake.reconcile()
|
||||
intake.arm()
|
||||
except Exception:
|
||||
if intake is not None:
|
||||
with suppress(Exception):
|
||||
intake.shutdown()
|
||||
self._state = "disarmed"
|
||||
return "conflict"
|
||||
assert intake is not None
|
||||
cancelled = Event()
|
||||
worker = Thread(
|
||||
target=self._drain_loop,
|
||||
args=(intake, cancelled),
|
||||
name="shd-capture-drain",
|
||||
daemon=True,
|
||||
)
|
||||
self._intake = intake
|
||||
self._cancelled = cancelled
|
||||
self._worker = worker
|
||||
self._state = "armed"
|
||||
try:
|
||||
worker.start()
|
||||
except Exception:
|
||||
self._state = "disarmed"
|
||||
self._intake = self._cancelled = self._worker = None
|
||||
with suppress(Exception):
|
||||
intake.shutdown()
|
||||
return "conflict"
|
||||
return "started"
|
||||
|
||||
def stop_session(self) -> ControlOutcome:
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
worker = self._worker
|
||||
if worker is None:
|
||||
return "already_stopped"
|
||||
if self._state == "stopping":
|
||||
return "conflict"
|
||||
self._state = "stopping"
|
||||
cancelled = self._cancelled
|
||||
intake = self._intake
|
||||
return "stopped" if self._stop_worker(worker, intake, cancelled) else "conflict"
|
||||
|
||||
def shutdown(self) -> bool:
|
||||
"""Request child shutdown before the service gives up its instance lock."""
|
||||
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
worker = self._worker
|
||||
if worker is None:
|
||||
return True
|
||||
self._state = "stopping"
|
||||
intake = self._intake
|
||||
cancelled = self._cancelled
|
||||
return self._stop_worker(worker, intake, cancelled)
|
||||
|
||||
def defer_resource_close(self, close: Callable[[], None]) -> None:
|
||||
"""Release the service lock only after a timed-out drain worker exits."""
|
||||
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
worker = self._worker
|
||||
if worker is None:
|
||||
close()
|
||||
return
|
||||
|
||||
def wait_for_worker() -> None:
|
||||
worker.join()
|
||||
close()
|
||||
|
||||
Thread(target=wait_for_worker, name="shd-capture-release", daemon=True).start()
|
||||
|
||||
def _stop_worker(
|
||||
self,
|
||||
worker: Thread,
|
||||
intake: IntakePort | None,
|
||||
cancelled: Event | None,
|
||||
) -> bool:
|
||||
if cancelled is not None:
|
||||
cancelled.set()
|
||||
timeout = float(self._runtime.config.lifecycle.shutdown_timeout_seconds)
|
||||
initial_timeout = timeout / 2
|
||||
# Never hold the control lock while waiting for a drain or child cleanup.
|
||||
worker.join(timeout=initial_timeout)
|
||||
if worker.is_alive():
|
||||
# A receipt in flight remains durable because cancellation prevents its
|
||||
# acknowledgement. Stopping intake unblocks watcher/helper cleanup.
|
||||
if intake is not None:
|
||||
with suppress(Exception):
|
||||
intake.shutdown()
|
||||
worker.join(timeout=timeout - initial_timeout)
|
||||
with self._lock:
|
||||
self._reap_finished_worker()
|
||||
return not worker.is_alive()
|
||||
|
||||
def _drain_loop(self, intake: IntakePort, cancelled: Event) -> None:
|
||||
try:
|
||||
while not cancelled.is_set():
|
||||
intake.tick()
|
||||
self._runtime._drain(intake, self._repository, cancelled.is_set)
|
||||
cancelled.wait(_POLL_SECONDS)
|
||||
except Exception:
|
||||
# The HTTP boundary exposes only bounded armed/queue state, not an
|
||||
# intake, storage, helper, or media exception.
|
||||
pass
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
intake.shutdown()
|
||||
with self._lock:
|
||||
if self._intake is intake:
|
||||
self._state = "disarmed"
|
||||
|
||||
def _reap_finished_worker(self) -> None:
|
||||
if self._worker is not None and not self._worker.is_alive():
|
||||
self._worker = self._cancelled = self._intake = None
|
||||
self._state = "disarmed"
|
||||
|
||||
def _queue_depth(self, intake: IntakePort | None) -> int:
|
||||
if intake is None:
|
||||
return 0
|
||||
depths: list[int] = []
|
||||
for name in ("queue_depth", "helper_queue_depth"):
|
||||
try:
|
||||
value = getattr(intake, name, 0)
|
||||
except Exception:
|
||||
value = 0
|
||||
if not isinstance(value, bool) and isinstance(value, int) and value >= 0:
|
||||
depths.append(value)
|
||||
return min(sum(depths), self._runtime.config.limits.max_queue_items)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _verified_png_file(path: Path, max_bytes: int) -> Iterator[Iterable[bytes]]:
|
||||
"""Yield a one-descriptor PNG stream and reject unsafe input without path disclosure."""
|
||||
|
||||
candidate = path.absolute()
|
||||
if len(str(candidate)) > 240:
|
||||
raise RuntimeFailure("input image is not a safe PNG")
|
||||
try:
|
||||
descriptor = os.open(candidate, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
except OSError as error:
|
||||
raise RuntimeFailure("input image is not a safe PNG") from error
|
||||
try:
|
||||
info = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(info.st_mode)
|
||||
or stat.S_IMODE(info.st_mode) & 0o022
|
||||
or not 8 <= info.st_size <= max_bytes
|
||||
):
|
||||
raise RuntimeFailure("input image is not a safe PNG")
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
first = os.read(descriptor, len(PNG_SIGNATURE))
|
||||
if first != PNG_SIGNATURE:
|
||||
raise RuntimeFailure("input image is not a safe PNG")
|
||||
total = len(first)
|
||||
yield first
|
||||
while chunk := os.read(descriptor, 64 * 1024):
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise RuntimeFailure("input image is not a safe PNG")
|
||||
yield chunk
|
||||
if total != info.st_size:
|
||||
raise RuntimeFailure("input image is not a safe PNG")
|
||||
|
||||
yield chunks()
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
class CaptureRuntime:
|
||||
"""Own utility-local storage, then stream verified media receipts into it."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AppConfig,
|
||||
paths: RuntimePaths,
|
||||
*,
|
||||
migrations_dir: Path | None = None,
|
||||
intake_factory: Callable[[WatchConfig], IntakePort] = ClipboardIntake,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.paths = paths
|
||||
self.migrations_dir = (
|
||||
Path(__file__).resolve().parents[2] / "migrations"
|
||||
if migrations_dir is None
|
||||
else migrations_dir
|
||||
)
|
||||
self.intake_factory = intake_factory
|
||||
|
||||
def _watch_config(self) -> WatchConfig:
|
||||
return WatchConfig(
|
||||
spool=self.paths.cache_dir / "spool",
|
||||
max_image_bytes=self.config.limits.max_image_bytes,
|
||||
max_queue_items=self.config.limits.max_queue_items,
|
||||
policy_version=_POLICY_VERSION,
|
||||
shutdown_timeout_seconds=float(min(30, self.config.lifecycle.shutdown_timeout_seconds)),
|
||||
)
|
||||
|
||||
def validate_plan(self) -> None:
|
||||
"""Validate static configuration and paths without creating runtime resources."""
|
||||
|
||||
self._watch_config()
|
||||
if not self.migrations_dir.is_absolute() or not self.migrations_dir.is_dir():
|
||||
raise RuntimeFailure("capture runtime resources are unavailable")
|
||||
StorageDatabase(self.paths.data_dir, self.migrations_dir)
|
||||
ContentAddressedBlobStore(self.paths.data_dir)
|
||||
|
||||
def _start(self) -> _Resources:
|
||||
self.validate_plan()
|
||||
prepare_private_directories(self.paths)
|
||||
lock = InstanceLock.acquire(
|
||||
self.paths.instance_lock_file, self.config.lifecycle.lock_timeout_seconds
|
||||
)
|
||||
try:
|
||||
database = StorageDatabase(self.paths.data_dir, self.migrations_dir)
|
||||
database.migrate()
|
||||
blobs = ContentAddressedBlobStore(self.paths.data_dir)
|
||||
blobs.reconcile()
|
||||
JobLeases(database).recover()
|
||||
corpus = CaptureCorpus(
|
||||
database,
|
||||
blobs,
|
||||
max_media_bytes=self.config.limits.max_image_bytes,
|
||||
)
|
||||
corpus.reconcile_deletions()
|
||||
return _Resources(CaptureRepository(database, blobs), corpus, lock)
|
||||
except RuntimeFailure:
|
||||
lock.release()
|
||||
raise
|
||||
except Exception as error:
|
||||
lock.release()
|
||||
raise RuntimeFailure("capture runtime initialization failed") from error
|
||||
|
||||
def ingest_image(self, image: Path) -> str:
|
||||
"""Persist one descriptor-verified PNG as a unique capture occurrence."""
|
||||
|
||||
resources = self._start()
|
||||
token = secrets.token_hex(16)
|
||||
try:
|
||||
with _verified_png_file(image, self.config.limits.max_image_bytes) as stream:
|
||||
return resources.repository.receive(
|
||||
f"capture-{token}",
|
||||
f"image-{token}",
|
||||
stream,
|
||||
media_type="image/png",
|
||||
max_bytes=self.config.limits.max_image_bytes,
|
||||
)
|
||||
except RuntimeFailure:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise RuntimeFailure("image capture could not be stored") from error
|
||||
finally:
|
||||
resources.close()
|
||||
|
||||
def _drain(
|
||||
self,
|
||||
intake: IntakePort,
|
||||
repository: CaptureRepository,
|
||||
cancellation_requested: Callable[[], bool],
|
||||
) -> None:
|
||||
while (
|
||||
not cancellation_requested() and (payload := intake.take_helper_receipt()) is not None
|
||||
):
|
||||
intake.accept_helper_receipt(
|
||||
payload,
|
||||
occurrence_id=f"capture-{secrets.token_hex(16)}",
|
||||
processing_version=_PROCESSING_VERSION,
|
||||
)
|
||||
while not cancellation_requested() and (receipt := intake.take()) is not None:
|
||||
with intake.open_verified(receipt) as reader:
|
||||
repository.receive(
|
||||
receipt.occurrence_id,
|
||||
receipt.receipt_token,
|
||||
reader,
|
||||
media_type="image/png",
|
||||
max_bytes=self.config.limits.max_image_bytes,
|
||||
)
|
||||
if cancellation_requested():
|
||||
return
|
||||
if not intake.acknowledge(receipt):
|
||||
raise RuntimeFailure("capture receipt acknowledgement failed")
|
||||
|
||||
def serve(self) -> None:
|
||||
"""Run one local web service with unarmed, durable capture control."""
|
||||
|
||||
bootstrap_token = new_bootstrap_token()
|
||||
resources = self._start()
|
||||
control = _CaptureControl(self, resources.repository)
|
||||
authority = (
|
||||
f"[{self.config.server.host}]"
|
||||
if ":" in self.config.server.host
|
||||
else self.config.server.host
|
||||
)
|
||||
try:
|
||||
app = create_app(
|
||||
config=WebConfig(
|
||||
server=self.config.server,
|
||||
max_capture_items=min(64, self.config.limits.max_queue_items),
|
||||
max_image_bytes=self.config.limits.max_image_bytes,
|
||||
),
|
||||
capture_control=control,
|
||||
corpus=resources.corpus,
|
||||
bootstrap_token=bootstrap_token,
|
||||
)
|
||||
print(
|
||||
f"Open local review service: http://{authority}:{self.config.server.port}/"
|
||||
f"#bootstrap={bootstrap_token}"
|
||||
)
|
||||
print("H1: capture is disarmed; use Start Session before taking a screenshot.")
|
||||
try:
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=self.config.server.host,
|
||||
port=self.config.server.port,
|
||||
reload=False,
|
||||
workers=1,
|
||||
proxy_headers=False,
|
||||
)
|
||||
except SystemExit as error:
|
||||
if error.code not in (None, 0):
|
||||
raise RuntimeFailure("capture service failed") from error
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
except RuntimeFailure:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise RuntimeFailure("capture service failed") from error
|
||||
finally:
|
||||
if control.shutdown():
|
||||
resources.close()
|
||||
else:
|
||||
control.defer_resource_close(resources.close)
|
||||
raise RuntimeFailure("capture service shutdown failed")
|
||||
|
||||
def watch(self, *, cancelled: Event | None = None, poll_seconds: float = _POLL_SECONDS) -> None:
|
||||
"""Run the armed intake until cancellation, signal, or keyboard interruption."""
|
||||
|
||||
if not 0 < poll_seconds <= 1:
|
||||
raise ValueError("invalid runtime poll interval")
|
||||
resources = self._start()
|
||||
intake: IntakePort | None = None
|
||||
stop_requested = Event()
|
||||
handlers: dict[signal.Signals, object] = {}
|
||||
|
||||
def request_stop(_signal_number: int, _frame: object) -> None:
|
||||
stop_requested.set()
|
||||
|
||||
try:
|
||||
if current_thread() is main_thread():
|
||||
for configured_signal in (signal.SIGINT, signal.SIGTERM):
|
||||
handlers[configured_signal] = signal.signal(configured_signal, request_stop)
|
||||
intake = self.intake_factory(self._watch_config())
|
||||
intake.reconcile()
|
||||
intake.arm()
|
||||
while not stop_requested.is_set() and (cancelled is None or not cancelled.is_set()):
|
||||
intake.tick()
|
||||
self._drain(
|
||||
intake,
|
||||
resources.repository,
|
||||
lambda: (
|
||||
stop_requested.is_set() or (cancelled is not None and cancelled.is_set())
|
||||
),
|
||||
)
|
||||
time.sleep(poll_seconds)
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
except RuntimeFailure:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise RuntimeFailure("capture watch failed") from error
|
||||
finally:
|
||||
if intake is not None:
|
||||
with suppress(OSError, RuntimeError):
|
||||
intake.shutdown()
|
||||
for restored_signal, handler in handlers.items():
|
||||
signal.signal(restored_signal, cast(signal.Handlers, handler))
|
||||
resources.close()
|
||||
Reference in New Issue
Block a user