546 lines
20 KiB
Python
546 lines
20 KiB
Python
"""Bounded lifecycle wiring for local capture intake and utility storage."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import os
|
|
import secrets
|
|
import signal
|
|
import stat
|
|
import time
|
|
from collections.abc import Callable, Iterable, Iterator, Mapping
|
|
from contextlib import AbstractContextManager, contextmanager, suppress
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from threading import Event, RLock, Thread, current_thread, main_thread
|
|
from typing import Protocol, cast
|
|
|
|
import uvicorn
|
|
|
|
from game_data_capture.config import AppConfig
|
|
from game_data_capture.ingest.clipboard import ClipboardIntake, IntakeReceipt, WatchConfig
|
|
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE
|
|
from game_data_capture.paths import RuntimePaths, prepare_private_directories, prepare_private_file
|
|
from game_data_capture.pipeline.jobs import JobLeases
|
|
from game_data_capture.storage import (
|
|
CaptureCorpus,
|
|
CaptureRepository,
|
|
ContentAddressedBlobStore,
|
|
StorageDatabase,
|
|
)
|
|
from game_data_capture.web.app import (
|
|
CaptureControlPort,
|
|
CaptureSessionStatus,
|
|
ControlOutcome,
|
|
WebConfig,
|
|
create_app,
|
|
)
|
|
from game_data_capture.web.security import new_bootstrap_token
|
|
|
|
_POLICY_VERSION = "capture-v1"
|
|
_PROCESSING_VERSION = "capture-v1"
|
|
_POLL_SECONDS = 0.1
|
|
|
|
|
|
class RuntimeFailure(RuntimeError):
|
|
"""A safe-to-display runtime failure that does not disclose local inputs."""
|
|
|
|
|
|
class IntakePort(Protocol):
|
|
"""The accepted media descriptor boundary consumed by this runtime."""
|
|
|
|
def reconcile(self) -> list[str]: ...
|
|
|
|
def arm(self) -> None: ...
|
|
|
|
def tick(self) -> None: ...
|
|
|
|
def take_helper_receipt(self) -> dict[str, object] | None: ...
|
|
|
|
def accept_helper_receipt(
|
|
self,
|
|
payload: Mapping[str, object],
|
|
*,
|
|
occurrence_id: str,
|
|
processing_version: str,
|
|
) -> bool: ...
|
|
|
|
def take(self) -> IntakeReceipt | None: ...
|
|
|
|
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: ...
|
|
|
|
def acknowledge(self, receipt: IntakeReceipt) -> bool: ...
|
|
|
|
def shutdown(self) -> None: ...
|
|
|
|
|
|
class InstanceLock:
|
|
"""Descriptor-held ownership that safely reuses an unlocked stale lock file."""
|
|
|
|
def __init__(self, file_descriptor: int) -> None:
|
|
self._file_descriptor: int | None = file_descriptor
|
|
|
|
@classmethod
|
|
def acquire(cls, path: Path, timeout_seconds: int) -> InstanceLock:
|
|
try:
|
|
prepare_private_file(path)
|
|
descriptor = os.open(path, os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC)
|
|
except OSError as error:
|
|
raise RuntimeFailure("capture instance lock is unavailable") from error
|
|
try:
|
|
info = os.fstat(descriptor)
|
|
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600:
|
|
raise RuntimeFailure("capture instance lock is unsafe")
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while True:
|
|
try:
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return cls(descriptor)
|
|
except BlockingIOError:
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeFailure("another capture instance is active") from None
|
|
time.sleep(min(0.05, max(0.0, deadline - time.monotonic())))
|
|
except Exception:
|
|
os.close(descriptor)
|
|
raise
|
|
|
|
def release(self) -> None:
|
|
if self._file_descriptor is None:
|
|
return
|
|
descriptor, self._file_descriptor = self._file_descriptor, None
|
|
with suppress(OSError):
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
with suppress(OSError):
|
|
os.close(descriptor)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _Resources:
|
|
repository: CaptureRepository
|
|
corpus: CaptureCorpus
|
|
lock: InstanceLock
|
|
|
|
def close(self) -> None:
|
|
self.lock.release()
|
|
|
|
|
|
class _CaptureControl(CaptureControlPort):
|
|
"""Serialize one optional armed intake beneath a running local service."""
|
|
|
|
def __init__(
|
|
self,
|
|
runtime: CaptureRuntime,
|
|
repository: CaptureRepository,
|
|
) -> None:
|
|
self._runtime = runtime
|
|
self._repository = repository
|
|
self._lock = RLock()
|
|
self._cancelled: Event | None = None
|
|
self._intake: IntakePort | None = None
|
|
self._worker: Thread | None = None
|
|
self._state = "disarmed"
|
|
|
|
def status(self) -> CaptureSessionStatus:
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
intake = self._intake
|
|
depth = self._queue_depth(intake)
|
|
return CaptureSessionStatus(
|
|
armed=self._state in {"armed", "stopping"},
|
|
queue_depth=depth,
|
|
queue_capacity=self._runtime.config.limits.max_queue_items,
|
|
)
|
|
|
|
def start_session(self) -> ControlOutcome:
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
if self._state == "armed":
|
|
return "already_armed"
|
|
if self._state == "stopping":
|
|
return "conflict"
|
|
if self._worker is not None:
|
|
return "conflict"
|
|
intake: IntakePort | None = None
|
|
try:
|
|
intake = self._runtime.intake_factory(self._runtime._watch_config())
|
|
# Durable media always becomes eligible before the watcher is armed.
|
|
intake.reconcile()
|
|
intake.arm()
|
|
except Exception:
|
|
if intake is not None:
|
|
with suppress(Exception):
|
|
intake.shutdown()
|
|
self._state = "disarmed"
|
|
return "conflict"
|
|
assert intake is not None
|
|
cancelled = Event()
|
|
worker = Thread(
|
|
target=self._drain_loop,
|
|
args=(intake, cancelled),
|
|
name="shd-capture-drain",
|
|
daemon=True,
|
|
)
|
|
self._intake = intake
|
|
self._cancelled = cancelled
|
|
self._worker = worker
|
|
self._state = "armed"
|
|
try:
|
|
worker.start()
|
|
except Exception:
|
|
self._state = "disarmed"
|
|
self._intake = self._cancelled = self._worker = None
|
|
with suppress(Exception):
|
|
intake.shutdown()
|
|
return "conflict"
|
|
return "started"
|
|
|
|
def stop_session(self) -> ControlOutcome:
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
worker = self._worker
|
|
if worker is None:
|
|
return "already_stopped"
|
|
if self._state == "stopping":
|
|
return "conflict"
|
|
self._state = "stopping"
|
|
cancelled = self._cancelled
|
|
intake = self._intake
|
|
return "stopped" if self._stop_worker(worker, intake, cancelled) else "conflict"
|
|
|
|
def shutdown(self) -> bool:
|
|
"""Request child shutdown before the service gives up its instance lock."""
|
|
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
worker = self._worker
|
|
if worker is None:
|
|
return True
|
|
self._state = "stopping"
|
|
intake = self._intake
|
|
cancelled = self._cancelled
|
|
return self._stop_worker(worker, intake, cancelled)
|
|
|
|
def defer_resource_close(self, close: Callable[[], None]) -> None:
|
|
"""Release the service lock only after a timed-out drain worker exits."""
|
|
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
worker = self._worker
|
|
if worker is None:
|
|
close()
|
|
return
|
|
|
|
def wait_for_worker() -> None:
|
|
worker.join()
|
|
close()
|
|
|
|
Thread(target=wait_for_worker, name="shd-capture-release", daemon=True).start()
|
|
|
|
def _stop_worker(
|
|
self,
|
|
worker: Thread,
|
|
intake: IntakePort | None,
|
|
cancelled: Event | None,
|
|
) -> bool:
|
|
if cancelled is not None:
|
|
cancelled.set()
|
|
timeout = float(self._runtime.config.lifecycle.shutdown_timeout_seconds)
|
|
initial_timeout = timeout / 2
|
|
# Never hold the control lock while waiting for a drain or child cleanup.
|
|
worker.join(timeout=initial_timeout)
|
|
if worker.is_alive():
|
|
# A receipt in flight remains durable because cancellation prevents its
|
|
# acknowledgement. Stopping intake unblocks watcher/helper cleanup.
|
|
if intake is not None:
|
|
with suppress(Exception):
|
|
intake.shutdown()
|
|
worker.join(timeout=timeout - initial_timeout)
|
|
with self._lock:
|
|
self._reap_finished_worker()
|
|
return not worker.is_alive()
|
|
|
|
def _drain_loop(self, intake: IntakePort, cancelled: Event) -> None:
|
|
try:
|
|
while not cancelled.is_set():
|
|
intake.tick()
|
|
self._runtime._drain(intake, self._repository, cancelled.is_set)
|
|
cancelled.wait(_POLL_SECONDS)
|
|
except Exception:
|
|
# The HTTP boundary exposes only bounded armed/queue state, not an
|
|
# intake, storage, helper, or media exception.
|
|
pass
|
|
finally:
|
|
with suppress(Exception):
|
|
intake.shutdown()
|
|
with self._lock:
|
|
if self._intake is intake:
|
|
self._state = "disarmed"
|
|
|
|
def _reap_finished_worker(self) -> None:
|
|
if self._worker is not None and not self._worker.is_alive():
|
|
self._worker = self._cancelled = self._intake = None
|
|
self._state = "disarmed"
|
|
|
|
def _queue_depth(self, intake: IntakePort | None) -> int:
|
|
if intake is None:
|
|
return 0
|
|
depths: list[int] = []
|
|
for name in ("queue_depth", "helper_queue_depth"):
|
|
try:
|
|
value = getattr(intake, name, 0)
|
|
except Exception:
|
|
value = 0
|
|
if not isinstance(value, bool) and isinstance(value, int) and value >= 0:
|
|
depths.append(value)
|
|
return min(sum(depths), self._runtime.config.limits.max_queue_items)
|
|
|
|
|
|
@contextmanager
|
|
def _verified_png_file(path: Path, max_bytes: int) -> Iterator[Iterable[bytes]]:
|
|
"""Yield a one-descriptor PNG stream and reject unsafe input without path disclosure."""
|
|
|
|
candidate = path.absolute()
|
|
if len(str(candidate)) > 240:
|
|
raise RuntimeFailure("input image is not a safe PNG")
|
|
try:
|
|
descriptor = os.open(candidate, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
|
except OSError as error:
|
|
raise RuntimeFailure("input image is not a safe PNG") from error
|
|
try:
|
|
info = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(info.st_mode)
|
|
or stat.S_IMODE(info.st_mode) & 0o022
|
|
or not 8 <= info.st_size <= max_bytes
|
|
):
|
|
raise RuntimeFailure("input image is not a safe PNG")
|
|
|
|
def chunks() -> Iterator[bytes]:
|
|
first = os.read(descriptor, len(PNG_SIGNATURE))
|
|
if first != PNG_SIGNATURE:
|
|
raise RuntimeFailure("input image is not a safe PNG")
|
|
total = len(first)
|
|
yield first
|
|
while chunk := os.read(descriptor, 64 * 1024):
|
|
total += len(chunk)
|
|
if total > max_bytes:
|
|
raise RuntimeFailure("input image is not a safe PNG")
|
|
yield chunk
|
|
if total != info.st_size:
|
|
raise RuntimeFailure("input image is not a safe PNG")
|
|
|
|
yield chunks()
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
class CaptureRuntime:
|
|
"""Own utility-local storage, then stream verified media receipts into it."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: AppConfig,
|
|
paths: RuntimePaths,
|
|
*,
|
|
migrations_dir: Path | None = None,
|
|
intake_factory: Callable[[WatchConfig], IntakePort] = ClipboardIntake,
|
|
) -> None:
|
|
self.config = config
|
|
self.paths = paths
|
|
self.migrations_dir = (
|
|
Path(__file__).resolve().parents[2] / "migrations"
|
|
if migrations_dir is None
|
|
else migrations_dir
|
|
)
|
|
self.intake_factory = intake_factory
|
|
|
|
def _watch_config(self) -> WatchConfig:
|
|
return WatchConfig(
|
|
spool=self.paths.cache_dir / "spool",
|
|
max_image_bytes=self.config.limits.max_image_bytes,
|
|
max_queue_items=self.config.limits.max_queue_items,
|
|
policy_version=_POLICY_VERSION,
|
|
shutdown_timeout_seconds=float(min(30, self.config.lifecycle.shutdown_timeout_seconds)),
|
|
)
|
|
|
|
def validate_plan(self) -> None:
|
|
"""Validate static configuration and paths without creating runtime resources."""
|
|
|
|
self._watch_config()
|
|
if not self.migrations_dir.is_absolute() or not self.migrations_dir.is_dir():
|
|
raise RuntimeFailure("capture runtime resources are unavailable")
|
|
StorageDatabase(self.paths.data_dir, self.migrations_dir)
|
|
ContentAddressedBlobStore(self.paths.data_dir)
|
|
|
|
def _start(self) -> _Resources:
|
|
self.validate_plan()
|
|
prepare_private_directories(self.paths)
|
|
lock = InstanceLock.acquire(
|
|
self.paths.instance_lock_file, self.config.lifecycle.lock_timeout_seconds
|
|
)
|
|
try:
|
|
database = StorageDatabase(self.paths.data_dir, self.migrations_dir)
|
|
database.migrate()
|
|
blobs = ContentAddressedBlobStore(self.paths.data_dir)
|
|
blobs.reconcile()
|
|
JobLeases(database).recover()
|
|
corpus = CaptureCorpus(
|
|
database,
|
|
blobs,
|
|
max_media_bytes=self.config.limits.max_image_bytes,
|
|
)
|
|
corpus.reconcile_deletions()
|
|
return _Resources(CaptureRepository(database, blobs), corpus, lock)
|
|
except RuntimeFailure:
|
|
lock.release()
|
|
raise
|
|
except Exception as error:
|
|
lock.release()
|
|
raise RuntimeFailure("capture runtime initialization failed") from error
|
|
|
|
def ingest_image(self, image: Path) -> str:
|
|
"""Persist one descriptor-verified PNG as a unique capture occurrence."""
|
|
|
|
resources = self._start()
|
|
token = secrets.token_hex(16)
|
|
try:
|
|
with _verified_png_file(image, self.config.limits.max_image_bytes) as stream:
|
|
return resources.repository.receive(
|
|
f"capture-{token}",
|
|
f"image-{token}",
|
|
stream,
|
|
media_type="image/png",
|
|
max_bytes=self.config.limits.max_image_bytes,
|
|
)
|
|
except RuntimeFailure:
|
|
raise
|
|
except Exception as error:
|
|
raise RuntimeFailure("image capture could not be stored") from error
|
|
finally:
|
|
resources.close()
|
|
|
|
def _drain(
|
|
self,
|
|
intake: IntakePort,
|
|
repository: CaptureRepository,
|
|
cancellation_requested: Callable[[], bool],
|
|
) -> None:
|
|
while (
|
|
not cancellation_requested() and (payload := intake.take_helper_receipt()) is not None
|
|
):
|
|
intake.accept_helper_receipt(
|
|
payload,
|
|
occurrence_id=f"capture-{secrets.token_hex(16)}",
|
|
processing_version=_PROCESSING_VERSION,
|
|
)
|
|
while not cancellation_requested() and (receipt := intake.take()) is not None:
|
|
with intake.open_verified(receipt) as reader:
|
|
repository.receive(
|
|
receipt.occurrence_id,
|
|
receipt.receipt_token,
|
|
reader,
|
|
media_type="image/png",
|
|
max_bytes=self.config.limits.max_image_bytes,
|
|
)
|
|
if cancellation_requested():
|
|
return
|
|
if not intake.acknowledge(receipt):
|
|
raise RuntimeFailure("capture receipt acknowledgement failed")
|
|
|
|
def serve(self) -> None:
|
|
"""Run one local web service with unarmed, durable capture control."""
|
|
|
|
bootstrap_token = new_bootstrap_token()
|
|
resources = self._start()
|
|
control = _CaptureControl(self, resources.repository)
|
|
authority = (
|
|
f"[{self.config.server.host}]"
|
|
if ":" in self.config.server.host
|
|
else self.config.server.host
|
|
)
|
|
try:
|
|
app = create_app(
|
|
config=WebConfig(
|
|
server=self.config.server,
|
|
max_capture_items=min(64, self.config.limits.max_queue_items),
|
|
max_image_bytes=self.config.limits.max_image_bytes,
|
|
),
|
|
capture_control=control,
|
|
corpus=resources.corpus,
|
|
bootstrap_token=bootstrap_token,
|
|
)
|
|
print(
|
|
f"Open local review service: http://{authority}:{self.config.server.port}/"
|
|
f"#bootstrap={bootstrap_token}"
|
|
)
|
|
print("H1: capture is disarmed; use Start Session before taking a screenshot.")
|
|
try:
|
|
uvicorn.run(
|
|
app,
|
|
host=self.config.server.host,
|
|
port=self.config.server.port,
|
|
reload=False,
|
|
workers=1,
|
|
proxy_headers=False,
|
|
)
|
|
except SystemExit as error:
|
|
if error.code not in (None, 0):
|
|
raise RuntimeFailure("capture service failed") from error
|
|
raise
|
|
except KeyboardInterrupt:
|
|
return
|
|
except RuntimeFailure:
|
|
raise
|
|
except Exception as error:
|
|
raise RuntimeFailure("capture service failed") from error
|
|
finally:
|
|
if control.shutdown():
|
|
resources.close()
|
|
else:
|
|
control.defer_resource_close(resources.close)
|
|
raise RuntimeFailure("capture service shutdown failed")
|
|
|
|
def watch(self, *, cancelled: Event | None = None, poll_seconds: float = _POLL_SECONDS) -> None:
|
|
"""Run the armed intake until cancellation, signal, or keyboard interruption."""
|
|
|
|
if not 0 < poll_seconds <= 1:
|
|
raise ValueError("invalid runtime poll interval")
|
|
resources = self._start()
|
|
intake: IntakePort | None = None
|
|
stop_requested = Event()
|
|
handlers: dict[signal.Signals, object] = {}
|
|
|
|
def request_stop(_signal_number: int, _frame: object) -> None:
|
|
stop_requested.set()
|
|
|
|
try:
|
|
if current_thread() is main_thread():
|
|
for configured_signal in (signal.SIGINT, signal.SIGTERM):
|
|
handlers[configured_signal] = signal.signal(configured_signal, request_stop)
|
|
intake = self.intake_factory(self._watch_config())
|
|
intake.reconcile()
|
|
intake.arm()
|
|
while not stop_requested.is_set() and (cancelled is None or not cancelled.is_set()):
|
|
intake.tick()
|
|
self._drain(
|
|
intake,
|
|
resources.repository,
|
|
lambda: (
|
|
stop_requested.is_set() or (cancelled is not None and cancelled.is_set())
|
|
),
|
|
)
|
|
time.sleep(poll_seconds)
|
|
except KeyboardInterrupt:
|
|
return
|
|
except RuntimeFailure:
|
|
raise
|
|
except Exception as error:
|
|
raise RuntimeFailure("capture watch failed") from error
|
|
finally:
|
|
if intake is not None:
|
|
with suppress(OSError, RuntimeError):
|
|
intake.shutdown()
|
|
for restored_signal, handler in handlers.items():
|
|
signal.signal(restored_signal, cast(signal.Handlers, handler))
|
|
resources.close()
|