Add game data capture vertical slice
This commit is contained in:
@@ -0,0 +1,479 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import game_data_capture.ingest.clipboard as clipboard
|
||||
from game_data_capture.ingest.clipboard import (
|
||||
ClipboardIntake,
|
||||
FocusedWindowMetadata,
|
||||
WatchConfig,
|
||||
focused_window,
|
||||
restricted_environment,
|
||||
)
|
||||
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, spool_png
|
||||
|
||||
|
||||
def receipt(spool: Path, token: str = "receipt") -> dict[str, object]:
|
||||
return spool_png(
|
||||
__import__("io").BytesIO(PNG_SIGNATURE + b"png"),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token=token,
|
||||
policy_version="v1",
|
||||
)
|
||||
|
||||
|
||||
def unique_receipt(spool: Path, token: str, value: bytes) -> dict[str, object]:
|
||||
return spool_png(
|
||||
io.BytesIO(PNG_SIGNATURE + value),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token=token,
|
||||
policy_version="v1",
|
||||
)
|
||||
|
||||
|
||||
def config(tmp_path: Path, *, queue: int = 2, retries: int = 2) -> WatchConfig:
|
||||
return WatchConfig((tmp_path / "spool").resolve(), 100, queue, "v1", retries, 0.01, 0.1)
|
||||
|
||||
|
||||
def test_environment_is_allowlisted_and_secret_free() -> None:
|
||||
child = restricted_environment(
|
||||
{
|
||||
"PATH": "/bin",
|
||||
"XDG_RUNTIME_DIR": "/run/user/1",
|
||||
"OPENAI_API_KEY": "sk-secret",
|
||||
"HTTP_AUTHORIZATION": "Bearer secret",
|
||||
"PYTHONPATH": "/unsafe",
|
||||
"UNRELATED": "no",
|
||||
}
|
||||
)
|
||||
assert child == {"PATH": "/bin", "XDG_RUNTIME_DIR": "/run/user/1"}
|
||||
|
||||
|
||||
def test_text_clear_duplicate_and_backpressure_are_silent(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
item = receipt(intake.config.spool)
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
assert not intake.accept_helper_receipt(
|
||||
{"media_type": "text/plain"}, occurrence_id="a", processing_version="p"
|
||||
)
|
||||
assert not intake.accept_helper_receipt({}, occurrence_id="a", processing_version="p")
|
||||
assert intake.accept_helper_receipt(
|
||||
item, occurrence_id="a", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id="a", processing_version="p", metadata=metadata
|
||||
)
|
||||
other = receipt(intake.config.spool, "other")
|
||||
assert not intake.accept_helper_receipt(
|
||||
other, occurrence_id="b", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.queue_depth == 1
|
||||
|
||||
|
||||
def test_watcher_argv_places_options_before_watch_subcommand(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
launched: list[list[str]] = []
|
||||
|
||||
class Fake:
|
||||
pid = os.getpid()
|
||||
stdout = None
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return None
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
return 0
|
||||
|
||||
def popen(args: list[str], **kwargs: object) -> Fake:
|
||||
assert kwargs["shell"] is False
|
||||
launched.append(args)
|
||||
return Fake()
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", popen)
|
||||
monkeypatch.setattr(os, "killpg", lambda _pid, _signal: None)
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.arm()
|
||||
command = launched[0]
|
||||
watch = command.index("--watch")
|
||||
assert command[:watch] == ["wl-paste", "--type", "image/png"]
|
||||
assert command[watch + 1] == sys.executable
|
||||
assert "--" not in command
|
||||
assert "game_data_capture.ingest.spool_helper" in command
|
||||
intake.pause()
|
||||
assert intake.paused
|
||||
intake.resume()
|
||||
assert len(launched) == 2
|
||||
intake.disarm()
|
||||
assert not intake.armed
|
||||
|
||||
|
||||
def test_cancellation_disarms_and_clears_queue(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
assert intake.accept_helper_receipt(
|
||||
receipt(intake.config.spool),
|
||||
occurrence_id="occurrence",
|
||||
processing_version="processing-v1",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
intake.cancel()
|
||||
assert not intake.armed and intake.take() is None
|
||||
|
||||
|
||||
def test_receipt_integrity_versions_and_bounded_duplicate_cache(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=2))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
first = receipt(intake.config.spool, "one")
|
||||
future = dict(first, receipt_version=2)
|
||||
assert not intake.accept_helper_receipt(
|
||||
future, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.accept_helper_receipt(
|
||||
first, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
first, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert len(intake._seen_tokens) == 1
|
||||
intake.shutdown()
|
||||
assert not intake._seen_tokens and not intake._helper_payloads
|
||||
|
||||
|
||||
def test_durable_receipt_recovery_is_null_metadata_duplicate_safe_and_acknowledged(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
item = receipt(intake.config.spool, "crash-token")
|
||||
intake.armed = True
|
||||
monkeypatch.setattr(
|
||||
clipboard,
|
||||
"focused_window",
|
||||
lambda: pytest.fail("recovered receipt must not query the current window"),
|
||||
)
|
||||
intake.reconcile()
|
||||
recovered = intake.take_helper_receipt()
|
||||
assert recovered == item
|
||||
assert intake.accept_helper_receipt(
|
||||
recovered, occurrence_id="recovered", processing_version="p"
|
||||
)
|
||||
accepted = intake.take()
|
||||
assert accepted is not None
|
||||
assert accepted.focused_window == FocusedWindowMetadata(None, None, None)
|
||||
assert not intake.accept_helper_receipt(item, occurrence_id="live-copy", processing_version="p")
|
||||
assert (intake.config.spool / "crash-token.receipt.json").exists()
|
||||
assert intake.acknowledge(accepted)
|
||||
assert not intake.acknowledge(accepted)
|
||||
assert not (intake.config.spool / "crash-token.receipt.json").exists()
|
||||
assert accepted.spool_path.exists()
|
||||
|
||||
|
||||
def test_reconcile_respects_helper_backpressure_and_retains_durable_receipts(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
first = unique_receipt(intake.config.spool, "first", b"first")
|
||||
second = unique_receipt(intake.config.spool, "second", b"second")
|
||||
intake.armed = True
|
||||
intake.reconcile()
|
||||
queued = intake.take_helper_receipt()
|
||||
assert queued in (first, second)
|
||||
assert intake.accept_helper_receipt(
|
||||
queued,
|
||||
occurrence_id="first",
|
||||
processing_version="p",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
remaining = second if queued == first else first
|
||||
assert (intake.config.spool / f"{remaining['receipt_token']}.receipt.json").exists()
|
||||
intake.reconcile()
|
||||
assert intake.take_helper_receipt() == remaining
|
||||
|
||||
|
||||
def test_unacknowledged_receipt_survives_a_fresh_intake(tmp_path: Path) -> None:
|
||||
initial = ClipboardIntake(config(tmp_path))
|
||||
item = receipt(initial.config.spool, "retry-token")
|
||||
restarted = ClipboardIntake(config(tmp_path))
|
||||
restarted.armed = True
|
||||
restarted.reconcile()
|
||||
replay = restarted.take_helper_receipt()
|
||||
assert replay == item
|
||||
assert restarted.accept_helper_receipt(replay, occurrence_id="retry", processing_version="p")
|
||||
accepted = restarted.take()
|
||||
assert accepted is not None
|
||||
assert (restarted.config.spool / "retry-token.receipt.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["max_image_bytes", "max_queue_items", "max_restarts"])
|
||||
def test_bool_limits_are_rejected(tmp_path: Path, field: str) -> None:
|
||||
values: dict[str, object] = dict(
|
||||
spool=(tmp_path / "spool").resolve(),
|
||||
max_image_bytes=100,
|
||||
max_queue_items=1,
|
||||
policy_version="v1",
|
||||
max_restarts=1,
|
||||
)
|
||||
values[field] = True
|
||||
with pytest.raises(ValueError):
|
||||
WatchConfig(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_watcher_crash_uses_backoff_and_restart_cap(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
starts = 0
|
||||
|
||||
class Dead:
|
||||
pid = os.getpid()
|
||||
stdout = None
|
||||
|
||||
def poll(self) -> int:
|
||||
return 1
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
return 1
|
||||
|
||||
def popen(*_args: object, **_kwargs: object) -> Dead:
|
||||
nonlocal starts
|
||||
starts += 1
|
||||
return Dead()
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", popen)
|
||||
intake = ClipboardIntake(config(tmp_path, retries=1))
|
||||
intake.arm()
|
||||
intake.tick(0)
|
||||
assert starts == 1
|
||||
intake.tick(0.02)
|
||||
assert starts == 2
|
||||
intake.tick(0.02)
|
||||
intake.tick(1)
|
||||
assert starts == 2
|
||||
|
||||
|
||||
def test_niri_timeout_and_malformed_json_are_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: None)
|
||||
assert focused_window() == FocusedWindowMetadata(None, None, None)
|
||||
monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: b"{broken")
|
||||
assert focused_window() == FocusedWindowMetadata(None, None, None)
|
||||
|
||||
|
||||
def test_receipt_mutation_matrix_and_file_integrity(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
base = unique_receipt(intake.config.spool, "token", b"base")
|
||||
for key, value in (
|
||||
("receipt_version", 2),
|
||||
("receipt_token", ""),
|
||||
("sha256", "0" * 64),
|
||||
("byte_size", 9),
|
||||
("media_type", "text/plain"),
|
||||
("policy_version", "other"),
|
||||
("spool_name", "wrong.png"),
|
||||
):
|
||||
changed = dict(base)
|
||||
changed[key] = value
|
||||
assert not intake.accept_helper_receipt(
|
||||
changed, occurrence_id="ok", processing_version="p", metadata=metadata
|
||||
)
|
||||
for value in ("", "bad value"):
|
||||
assert not intake.accept_helper_receipt(
|
||||
base, occurrence_id=value, processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
base, occurrence_id="ok", processing_version=value, metadata=metadata
|
||||
)
|
||||
for index, (mode, content) in enumerate(
|
||||
((0o600, b"changed"), (0o600, PNG_SIGNATURE), (0o644, None))
|
||||
):
|
||||
item = unique_receipt(
|
||||
intake.config.spool, f"token{mode}{index}", bytes(f"{mode}{index}", "ascii")
|
||||
)
|
||||
path = intake.config.spool / str(item["spool_name"])
|
||||
if content is not None:
|
||||
path.write_bytes(content)
|
||||
path.chmod(mode)
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id=f"o{mode}", processing_version="p", metadata=metadata
|
||||
)
|
||||
item = unique_receipt(intake.config.spool, "symlink", b"link")
|
||||
path = intake.config.spool / str(item["spool_name"])
|
||||
path.unlink()
|
||||
path.symlink_to(tmp_path / "outside")
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id="symlink", processing_version="p", metadata=metadata
|
||||
)
|
||||
|
||||
|
||||
def test_open_verified_and_acceptance_close_descriptors(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
item = unique_receipt(intake.config.spool, "fd", b"reader")
|
||||
before = len(list(Path("/proc/self/fd").iterdir()))
|
||||
assert intake.accept_helper_receipt(
|
||||
item,
|
||||
occurrence_id="fd",
|
||||
processing_version="p",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
accepted = intake.take()
|
||||
assert accepted is not None and len(list(Path("/proc/self/fd").iterdir())) == before
|
||||
with intake.open_verified(accepted) as reader:
|
||||
assert b"".join(reader) == PNG_SIGNATURE + b"reader"
|
||||
path = accepted.spool_path
|
||||
path.write_bytes(b"changed")
|
||||
with pytest.raises(ValueError), intake.open_verified(accepted):
|
||||
pass
|
||||
|
||||
|
||||
def test_duplicate_cache_eviction_shutdown_and_unreferenced_candidates(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=2))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
items = [unique_receipt(intake.config.spool, f"t{i}", bytes([i])) for i in range(3)]
|
||||
for index, item in enumerate(items):
|
||||
assert intake.accept_helper_receipt(
|
||||
item, occurrence_id=f"o{index}", processing_version="p", metadata=metadata
|
||||
)
|
||||
intake.take()
|
||||
assert len(intake._seen_tokens) == 2 and not intake.accept_helper_receipt(
|
||||
items[2], occurrence_id="latest", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.accept_helper_receipt(
|
||||
items[0], occurrence_id="evicted", processing_version="p", metadata=metadata
|
||||
)
|
||||
intake = ClipboardIntake(config(tmp_path / "other", queue=1))
|
||||
intake.armed = True
|
||||
queued = unique_receipt(intake.config.spool, "queued", b"queued")
|
||||
extra = unique_receipt(intake.config.spool, "extra", b"extra")
|
||||
assert intake.accept_helper_receipt(
|
||||
queued, occurrence_id="queued", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert [path.name for path in intake.unreferenced_spool_candidates()] == [
|
||||
str(extra["spool_name"])
|
||||
]
|
||||
intake._helper_payloads.append({"receipt": "real"})
|
||||
intake.shutdown()
|
||||
assert (
|
||||
not intake._queue
|
||||
and not intake._helper_payloads
|
||||
and not intake._seen_tokens
|
||||
and not intake._seen_order
|
||||
)
|
||||
|
||||
|
||||
def test_receipt_reader_bounds_spawn_failures_and_config_types(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
class Output:
|
||||
def __init__(self) -> None:
|
||||
self.parts = [
|
||||
b"x" * 1025 + b"\n",
|
||||
json.dumps({"ok": 1}).encode() + b"\n",
|
||||
json.dumps({"two": 2}).encode() + b"\n",
|
||||
]
|
||||
|
||||
def readline(self, _size: int) -> bytes:
|
||||
return self.parts.pop(0) if self.parts else b""
|
||||
|
||||
class Process:
|
||||
stdout = Output()
|
||||
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
intake._read_receipts(Process()) # type: ignore[arg-type]
|
||||
assert intake.helper_queue_depth == 1 and intake.take_helper_receipt() == {"ok": 1}
|
||||
attempts = 0
|
||||
|
||||
def failing(*_args: object, **_kwargs: object) -> object:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
raise OSError("missing")
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", failing)
|
||||
failed = ClipboardIntake(config(tmp_path, retries=2))
|
||||
failed.arm()
|
||||
failed.tick(failed._restart_after + 0.01)
|
||||
failed.tick(failed._restart_after + 0.01)
|
||||
failed.tick(failed._restart_after + 1)
|
||||
assert attempts == 3
|
||||
for field, value in (
|
||||
("max_image_bytes", 1.5),
|
||||
("max_queue_items", "1"),
|
||||
("max_restarts", 1.5),
|
||||
("shutdown_timeout_seconds", 0),
|
||||
("restart_backoff_seconds", math.inf),
|
||||
):
|
||||
values: dict[str, object] = dict(
|
||||
spool=(tmp_path / "x").resolve(),
|
||||
max_image_bytes=100,
|
||||
max_queue_items=1,
|
||||
policy_version="v1",
|
||||
max_restarts=1,
|
||||
)
|
||||
values[field] = value
|
||||
with pytest.raises(ValueError):
|
||||
WatchConfig(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_bounded_niri_runner_and_valid_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert (
|
||||
clipboard._bounded_command_output([sys.executable, "-c", "print('x'*20000)"], {}, 1) is None
|
||||
)
|
||||
assert (
|
||||
clipboard._bounded_command_output(
|
||||
[sys.executable, "-c", "import time; time.sleep(2)"], {}, 0.01
|
||||
)
|
||||
is None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
clipboard,
|
||||
"_bounded_command_output",
|
||||
lambda *_args: b'{"app_id":"game","title":"title","id":1}',
|
||||
)
|
||||
assert focused_window() == FocusedWindowMetadata("game", "title", 1)
|
||||
|
||||
|
||||
def test_exited_watcher_is_waited_stdout_closed_and_reader_retired(tmp_path: Path) -> None:
|
||||
class Output:
|
||||
closed = False
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
class Child:
|
||||
pid = os.getpid()
|
||||
stdout = Output()
|
||||
waits = 0
|
||||
|
||||
def poll(self) -> int:
|
||||
return 1
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self.waits += 1
|
||||
return 1
|
||||
|
||||
class Reader:
|
||||
joined = False
|
||||
|
||||
def join(self, timeout: float | None = None) -> None:
|
||||
self.joined = True
|
||||
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
child = Child()
|
||||
reader = Reader()
|
||||
intake._watcher = child # type: ignore[assignment]
|
||||
intake._reader = reader # type: ignore[assignment]
|
||||
intake.pause()
|
||||
assert child.waits == 1 and child.stdout.closed and reader.joined and intake._reader is None
|
||||
@@ -0,0 +1,389 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import game_data_capture.ingest.spool_helper as spool_helper
|
||||
from game_data_capture.ingest.spool_helper import (
|
||||
PNG_SIGNATURE,
|
||||
SpoolError,
|
||||
acknowledge_receipt,
|
||||
durable_receipts,
|
||||
receipt_name,
|
||||
reconcile_spool,
|
||||
spool_png,
|
||||
)
|
||||
|
||||
|
||||
def png(payload: bytes = b"data") -> bytes:
|
||||
return PNG_SIGNATURE + payload
|
||||
|
||||
|
||||
def fill_receipt_capacity(spool: Path) -> dict[str, object]:
|
||||
seed = spool_png(
|
||||
io.BytesIO(png(b"seed")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="seed",
|
||||
policy_version="v1",
|
||||
)
|
||||
for index in range(255):
|
||||
token = f"pre{index}"
|
||||
sidecar = spool / receipt_name(token)
|
||||
sidecar.write_text(
|
||||
json.dumps(dict(seed, receipt_token=token), sort_keys=True, separators=(",", ":"))
|
||||
)
|
||||
sidecar.chmod(0o600)
|
||||
return seed
|
||||
|
||||
|
||||
def test_png_spool_is_private_atomic_and_deduplicated(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
replaced: list[tuple[Path, Path]] = []
|
||||
real_replace = os.replace
|
||||
|
||||
def replace(source: Path, destination: Path) -> None:
|
||||
replaced.append((source, destination))
|
||||
real_replace(source, destination)
|
||||
|
||||
monkeypatch.setattr(os, "replace", replace)
|
||||
first = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
second = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="two", policy_version="v1"
|
||||
)
|
||||
assert first["sha256"] == second["sha256"]
|
||||
assert (spool.stat().st_mode & 0o777) == 0o700
|
||||
installed = spool / str(first["spool_name"])
|
||||
assert installed.read_bytes() == png() and (installed.stat().st_mode & 0o777) == 0o600
|
||||
assert len(replaced) == 1 and replaced[0][1] == installed
|
||||
sidecars = sorted(spool.glob("*.receipt.json"))
|
||||
assert [sidecar.name for sidecar in sidecars] == ["one.receipt.json", "two.receipt.json"]
|
||||
assert all((sidecar.stat().st_mode & 0o777) == 0o600 for sidecar in sidecars)
|
||||
assert all(set(json.loads(sidecar.read_text())) == set(first) for sidecar in sidecars)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [b"", b"text", PNG_SIGNATURE[:4]])
|
||||
def test_empty_text_and_malformed_png_leave_no_partial(tmp_path: Path, value: bytes) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(value), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
def test_unsupported_mime_leaves_no_partial(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
media_type="text/plain",
|
||||
)
|
||||
assert not spool.exists()
|
||||
|
||||
|
||||
def test_short_reads_accumulate_the_png_signature(tmp_path: Path) -> None:
|
||||
class ShortReads:
|
||||
def __init__(self) -> None:
|
||||
self.parts = [PNG_SIGNATURE[:3], PNG_SIGNATURE[3:] + b"payload", b""]
|
||||
|
||||
def read(self, _size: int) -> bytes:
|
||||
return self.parts.pop(0)
|
||||
|
||||
receipt = spool_png(
|
||||
ShortReads(),
|
||||
spool=(tmp_path / "spool").resolve(),
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert receipt["byte_size"] == len(PNG_SIGNATURE + b"payload")
|
||||
|
||||
|
||||
def test_size_bound_and_crash_reconciliation_remove_partials(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"x" * 100)),
|
||||
spool=spool,
|
||||
max_bytes=10,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
leftover = spool / ".crash.partial"
|
||||
leftover.write_bytes(b"incomplete")
|
||||
assert reconcile_spool(spool) == [".crash.partial"]
|
||||
assert not leftover.exists()
|
||||
|
||||
|
||||
def test_pending_receipt_is_promoted_only_when_its_png_is_verified(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
receipt = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
final = spool / receipt_name("one")
|
||||
pending = spool / ".one.receipt.pending"
|
||||
os.replace(final, pending)
|
||||
assert reconcile_spool(spool) == []
|
||||
assert final.exists() and not pending.exists()
|
||||
final.unlink()
|
||||
pending.write_text(json.dumps(receipt))
|
||||
assert reconcile_spool(spool) == [pending.name]
|
||||
assert not pending.exists()
|
||||
|
||||
|
||||
def test_receipt_token_replay_cannot_replace_the_original_sidecar(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
original = spool_png(
|
||||
io.BytesIO(png(b"original")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
final = spool / receipt_name("same-token")
|
||||
original_bytes = final.read_bytes()
|
||||
assert (
|
||||
spool_png(
|
||||
io.BytesIO(png(b"original")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
== original
|
||||
)
|
||||
with pytest.raises(SpoolError, match="receipt token conflict"):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"different")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert final.read_bytes() == original_bytes
|
||||
conflicting = dict(original, sha256="0" * 64, spool_name=f"{'0' * 64}.png")
|
||||
pending = spool / ".same-token.receipt.pending"
|
||||
pending.write_text(json.dumps(conflicting, sort_keys=True, separators=(",", ":")))
|
||||
pending.chmod(0o600)
|
||||
assert pending.name in reconcile_spool(spool)
|
||||
assert final.read_bytes() == original_bytes and not pending.exists()
|
||||
|
||||
|
||||
def test_sidecars_are_bounded_validated_and_acknowledged_exactly_once(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
first = spool_png(
|
||||
io.BytesIO(png(b"one")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
second = spool_png(
|
||||
io.BytesIO(png(b"two")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="two",
|
||||
policy_version="v1",
|
||||
)
|
||||
malformed = spool / "bad.receipt.json"
|
||||
malformed.write_bytes(b"{" + b"x" * 1024)
|
||||
malformed.chmod(0o600)
|
||||
linked = spool / "linked.receipt.json"
|
||||
linked.symlink_to(malformed)
|
||||
assert {item["receipt_token"] for item in durable_receipts(spool, max_items=2)} == {
|
||||
"one",
|
||||
"two",
|
||||
}
|
||||
assert acknowledge_receipt(spool, first)
|
||||
assert not acknowledge_receipt(spool, first)
|
||||
assert (spool / receipt_name("two")).exists()
|
||||
assert not acknowledge_receipt(spool, dict(second, receipt_token="one"))
|
||||
|
||||
|
||||
def test_malformed_sidecars_do_not_starve_valid_receipts_or_partial_cleanup(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
for index in range(256):
|
||||
malformed = spool / f"{index:03d}.receipt.json"
|
||||
malformed.write_bytes(b"{")
|
||||
malformed.chmod(0o600)
|
||||
valid = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="valid", policy_version="v1"
|
||||
)
|
||||
assert list(durable_receipts(spool, max_items=1)) == [valid]
|
||||
assert list(spool.glob("*.receipt.json")) == [spool / "valid.receipt.json"]
|
||||
for index in range(256):
|
||||
pending = spool / f".pending{index}.receipt.pending"
|
||||
pending.write_text(
|
||||
json.dumps(
|
||||
dict(valid, receipt_token=f"pending{index}"), sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
)
|
||||
pending.chmod(0o600)
|
||||
partial = spool / ".after-pending.partial"
|
||||
partial.write_bytes(b"incomplete")
|
||||
assert partial.name in reconcile_spool(spool)
|
||||
assert not partial.exists()
|
||||
|
||||
|
||||
def test_receipt_capacity_rejects_new_tokens_but_accepts_exact_replay(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
seed = fill_receipt_capacity(spool)
|
||||
evidence = (spool / receipt_name("seed")).read_bytes()
|
||||
png_names = {path.name for path in spool.glob("*.png")}
|
||||
with pytest.raises(SpoolError, match="capacity"):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"new")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="new",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert (spool / receipt_name("seed")).read_bytes() == evidence
|
||||
assert {path.name for path in spool.glob("*.png")} == png_names
|
||||
assert (
|
||||
spool_png(
|
||||
io.BytesIO(png(b"seed")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="seed",
|
||||
policy_version="v1",
|
||||
)
|
||||
== seed
|
||||
)
|
||||
|
||||
|
||||
def test_receipt_capacity_reconciles_malformed_files_before_counting(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
for index in range(256):
|
||||
malformed = spool / f"bad{index}.receipt.json"
|
||||
malformed.write_bytes(b"{")
|
||||
malformed.chmod(0o600)
|
||||
accepted = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="accepted", policy_version="v1"
|
||||
)
|
||||
assert accepted["receipt_token"] == "accepted"
|
||||
assert list(spool.glob("bad*.receipt.json")) == []
|
||||
|
||||
|
||||
def test_receipt_capacity_is_serialized_across_helper_processes(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
fill_receipt_capacity(spool)
|
||||
for index in range(1):
|
||||
(spool / receipt_name(f"pre{index}")).unlink()
|
||||
arguments = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"game_data_capture.ingest.spool_helper",
|
||||
"--spool",
|
||||
str(spool),
|
||||
"--max-bytes",
|
||||
"100",
|
||||
"--policy-version",
|
||||
"v1",
|
||||
]
|
||||
first = subprocess.Popen([*arguments, "--receipt-token", "race-one"], stdin=subprocess.PIPE)
|
||||
second = subprocess.Popen([*arguments, "--receipt-token", "race-two"], stdin=subprocess.PIPE)
|
||||
assert first.stdin is not None and second.stdin is not None
|
||||
first.stdin.write(png(b"one"))
|
||||
first.stdin.close()
|
||||
second.stdin.write(png(b"two"))
|
||||
second.stdin.close()
|
||||
assert sorted((first.wait(), second.wait())) == [0, 2]
|
||||
assert len(list(spool.glob("*.receipt.json"))) == 256
|
||||
|
||||
|
||||
def test_spool_lock_is_private_rejects_links_and_releases_on_failure(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
(spool / ".spool.lock").symlink_to(tmp_path / "outside")
|
||||
with pytest.raises(SpoolError, match="unsafe spool lock"):
|
||||
spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="link", policy_version="v1"
|
||||
)
|
||||
(spool / ".spool.lock").unlink()
|
||||
with monkeypatch.context() as context:
|
||||
context.setattr(
|
||||
spool_helper,
|
||||
"_install_pending_sidecar",
|
||||
lambda *_args: (_ for _ in ()).throw(OSError()),
|
||||
)
|
||||
with pytest.raises(OSError):
|
||||
spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="failure",
|
||||
policy_version="v1",
|
||||
)
|
||||
accepted = spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="recovered",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert accepted["receipt_token"] == "recovered"
|
||||
assert (spool / ".spool.lock").stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_short_write_removes_partial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
monkeypatch.setattr(os, "write", lambda _fd, _data: 0)
|
||||
with pytest.raises(OSError):
|
||||
spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
def test_helper_source_has_no_service_or_storage_imports() -> None:
|
||||
source = Path(__file__).parents[3] / "src/game_data_capture/ingest/spool_helper.py"
|
||||
text = source.read_text(encoding="utf-8")
|
||||
for forbidden in ("fastapi", "sqlite", "openai", "game_data_capture.storage"):
|
||||
assert forbidden not in text.lower()
|
||||
|
||||
|
||||
def test_stdin_entrypoint_returns_only_bounded_json_receipt(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"game_data_capture.ingest.spool_helper",
|
||||
"--spool",
|
||||
str(spool),
|
||||
"--max-bytes",
|
||||
"100",
|
||||
"--receipt-token",
|
||||
"receipt",
|
||||
"--policy-version",
|
||||
"v1",
|
||||
],
|
||||
input=png(),
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env={"PATH": os.environ.get("PATH", "")},
|
||||
)
|
||||
assert completed.returncode == 0 and len(completed.stdout) <= 1024 and not completed.stderr
|
||||
receipt = json.loads(completed.stdout)
|
||||
assert receipt["media_type"] == "image/png" and receipt["receipt_token"] == "receipt"
|
||||
Reference in New Issue
Block a user