Add game data capture vertical slice
This commit is contained in:
@@ -0,0 +1,479 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import game_data_capture.ingest.clipboard as clipboard
|
||||
from game_data_capture.ingest.clipboard import (
|
||||
ClipboardIntake,
|
||||
FocusedWindowMetadata,
|
||||
WatchConfig,
|
||||
focused_window,
|
||||
restricted_environment,
|
||||
)
|
||||
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, spool_png
|
||||
|
||||
|
||||
def receipt(spool: Path, token: str = "receipt") -> dict[str, object]:
|
||||
return spool_png(
|
||||
__import__("io").BytesIO(PNG_SIGNATURE + b"png"),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token=token,
|
||||
policy_version="v1",
|
||||
)
|
||||
|
||||
|
||||
def unique_receipt(spool: Path, token: str, value: bytes) -> dict[str, object]:
|
||||
return spool_png(
|
||||
io.BytesIO(PNG_SIGNATURE + value),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token=token,
|
||||
policy_version="v1",
|
||||
)
|
||||
|
||||
|
||||
def config(tmp_path: Path, *, queue: int = 2, retries: int = 2) -> WatchConfig:
|
||||
return WatchConfig((tmp_path / "spool").resolve(), 100, queue, "v1", retries, 0.01, 0.1)
|
||||
|
||||
|
||||
def test_environment_is_allowlisted_and_secret_free() -> None:
|
||||
child = restricted_environment(
|
||||
{
|
||||
"PATH": "/bin",
|
||||
"XDG_RUNTIME_DIR": "/run/user/1",
|
||||
"OPENAI_API_KEY": "sk-secret",
|
||||
"HTTP_AUTHORIZATION": "Bearer secret",
|
||||
"PYTHONPATH": "/unsafe",
|
||||
"UNRELATED": "no",
|
||||
}
|
||||
)
|
||||
assert child == {"PATH": "/bin", "XDG_RUNTIME_DIR": "/run/user/1"}
|
||||
|
||||
|
||||
def test_text_clear_duplicate_and_backpressure_are_silent(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
item = receipt(intake.config.spool)
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
assert not intake.accept_helper_receipt(
|
||||
{"media_type": "text/plain"}, occurrence_id="a", processing_version="p"
|
||||
)
|
||||
assert not intake.accept_helper_receipt({}, occurrence_id="a", processing_version="p")
|
||||
assert intake.accept_helper_receipt(
|
||||
item, occurrence_id="a", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id="a", processing_version="p", metadata=metadata
|
||||
)
|
||||
other = receipt(intake.config.spool, "other")
|
||||
assert not intake.accept_helper_receipt(
|
||||
other, occurrence_id="b", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.queue_depth == 1
|
||||
|
||||
|
||||
def test_watcher_argv_places_options_before_watch_subcommand(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
launched: list[list[str]] = []
|
||||
|
||||
class Fake:
|
||||
pid = os.getpid()
|
||||
stdout = None
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return None
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
return 0
|
||||
|
||||
def popen(args: list[str], **kwargs: object) -> Fake:
|
||||
assert kwargs["shell"] is False
|
||||
launched.append(args)
|
||||
return Fake()
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", popen)
|
||||
monkeypatch.setattr(os, "killpg", lambda _pid, _signal: None)
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.arm()
|
||||
command = launched[0]
|
||||
watch = command.index("--watch")
|
||||
assert command[:watch] == ["wl-paste", "--type", "image/png"]
|
||||
assert command[watch + 1] == sys.executable
|
||||
assert "--" not in command
|
||||
assert "game_data_capture.ingest.spool_helper" in command
|
||||
intake.pause()
|
||||
assert intake.paused
|
||||
intake.resume()
|
||||
assert len(launched) == 2
|
||||
intake.disarm()
|
||||
assert not intake.armed
|
||||
|
||||
|
||||
def test_cancellation_disarms_and_clears_queue(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
assert intake.accept_helper_receipt(
|
||||
receipt(intake.config.spool),
|
||||
occurrence_id="occurrence",
|
||||
processing_version="processing-v1",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
intake.cancel()
|
||||
assert not intake.armed and intake.take() is None
|
||||
|
||||
|
||||
def test_receipt_integrity_versions_and_bounded_duplicate_cache(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=2))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
first = receipt(intake.config.spool, "one")
|
||||
future = dict(first, receipt_version=2)
|
||||
assert not intake.accept_helper_receipt(
|
||||
future, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.accept_helper_receipt(
|
||||
first, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
first, occurrence_id="one", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert len(intake._seen_tokens) == 1
|
||||
intake.shutdown()
|
||||
assert not intake._seen_tokens and not intake._helper_payloads
|
||||
|
||||
|
||||
def test_durable_receipt_recovery_is_null_metadata_duplicate_safe_and_acknowledged(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
item = receipt(intake.config.spool, "crash-token")
|
||||
intake.armed = True
|
||||
monkeypatch.setattr(
|
||||
clipboard,
|
||||
"focused_window",
|
||||
lambda: pytest.fail("recovered receipt must not query the current window"),
|
||||
)
|
||||
intake.reconcile()
|
||||
recovered = intake.take_helper_receipt()
|
||||
assert recovered == item
|
||||
assert intake.accept_helper_receipt(
|
||||
recovered, occurrence_id="recovered", processing_version="p"
|
||||
)
|
||||
accepted = intake.take()
|
||||
assert accepted is not None
|
||||
assert accepted.focused_window == FocusedWindowMetadata(None, None, None)
|
||||
assert not intake.accept_helper_receipt(item, occurrence_id="live-copy", processing_version="p")
|
||||
assert (intake.config.spool / "crash-token.receipt.json").exists()
|
||||
assert intake.acknowledge(accepted)
|
||||
assert not intake.acknowledge(accepted)
|
||||
assert not (intake.config.spool / "crash-token.receipt.json").exists()
|
||||
assert accepted.spool_path.exists()
|
||||
|
||||
|
||||
def test_reconcile_respects_helper_backpressure_and_retains_durable_receipts(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
first = unique_receipt(intake.config.spool, "first", b"first")
|
||||
second = unique_receipt(intake.config.spool, "second", b"second")
|
||||
intake.armed = True
|
||||
intake.reconcile()
|
||||
queued = intake.take_helper_receipt()
|
||||
assert queued in (first, second)
|
||||
assert intake.accept_helper_receipt(
|
||||
queued,
|
||||
occurrence_id="first",
|
||||
processing_version="p",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
remaining = second if queued == first else first
|
||||
assert (intake.config.spool / f"{remaining['receipt_token']}.receipt.json").exists()
|
||||
intake.reconcile()
|
||||
assert intake.take_helper_receipt() == remaining
|
||||
|
||||
|
||||
def test_unacknowledged_receipt_survives_a_fresh_intake(tmp_path: Path) -> None:
|
||||
initial = ClipboardIntake(config(tmp_path))
|
||||
item = receipt(initial.config.spool, "retry-token")
|
||||
restarted = ClipboardIntake(config(tmp_path))
|
||||
restarted.armed = True
|
||||
restarted.reconcile()
|
||||
replay = restarted.take_helper_receipt()
|
||||
assert replay == item
|
||||
assert restarted.accept_helper_receipt(replay, occurrence_id="retry", processing_version="p")
|
||||
accepted = restarted.take()
|
||||
assert accepted is not None
|
||||
assert (restarted.config.spool / "retry-token.receipt.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["max_image_bytes", "max_queue_items", "max_restarts"])
|
||||
def test_bool_limits_are_rejected(tmp_path: Path, field: str) -> None:
|
||||
values: dict[str, object] = dict(
|
||||
spool=(tmp_path / "spool").resolve(),
|
||||
max_image_bytes=100,
|
||||
max_queue_items=1,
|
||||
policy_version="v1",
|
||||
max_restarts=1,
|
||||
)
|
||||
values[field] = True
|
||||
with pytest.raises(ValueError):
|
||||
WatchConfig(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_watcher_crash_uses_backoff_and_restart_cap(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
starts = 0
|
||||
|
||||
class Dead:
|
||||
pid = os.getpid()
|
||||
stdout = None
|
||||
|
||||
def poll(self) -> int:
|
||||
return 1
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
return 1
|
||||
|
||||
def popen(*_args: object, **_kwargs: object) -> Dead:
|
||||
nonlocal starts
|
||||
starts += 1
|
||||
return Dead()
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", popen)
|
||||
intake = ClipboardIntake(config(tmp_path, retries=1))
|
||||
intake.arm()
|
||||
intake.tick(0)
|
||||
assert starts == 1
|
||||
intake.tick(0.02)
|
||||
assert starts == 2
|
||||
intake.tick(0.02)
|
||||
intake.tick(1)
|
||||
assert starts == 2
|
||||
|
||||
|
||||
def test_niri_timeout_and_malformed_json_are_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: None)
|
||||
assert focused_window() == FocusedWindowMetadata(None, None, None)
|
||||
monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: b"{broken")
|
||||
assert focused_window() == FocusedWindowMetadata(None, None, None)
|
||||
|
||||
|
||||
def test_receipt_mutation_matrix_and_file_integrity(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
base = unique_receipt(intake.config.spool, "token", b"base")
|
||||
for key, value in (
|
||||
("receipt_version", 2),
|
||||
("receipt_token", ""),
|
||||
("sha256", "0" * 64),
|
||||
("byte_size", 9),
|
||||
("media_type", "text/plain"),
|
||||
("policy_version", "other"),
|
||||
("spool_name", "wrong.png"),
|
||||
):
|
||||
changed = dict(base)
|
||||
changed[key] = value
|
||||
assert not intake.accept_helper_receipt(
|
||||
changed, occurrence_id="ok", processing_version="p", metadata=metadata
|
||||
)
|
||||
for value in ("", "bad value"):
|
||||
assert not intake.accept_helper_receipt(
|
||||
base, occurrence_id=value, processing_version="p", metadata=metadata
|
||||
)
|
||||
assert not intake.accept_helper_receipt(
|
||||
base, occurrence_id="ok", processing_version=value, metadata=metadata
|
||||
)
|
||||
for index, (mode, content) in enumerate(
|
||||
((0o600, b"changed"), (0o600, PNG_SIGNATURE), (0o644, None))
|
||||
):
|
||||
item = unique_receipt(
|
||||
intake.config.spool, f"token{mode}{index}", bytes(f"{mode}{index}", "ascii")
|
||||
)
|
||||
path = intake.config.spool / str(item["spool_name"])
|
||||
if content is not None:
|
||||
path.write_bytes(content)
|
||||
path.chmod(mode)
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id=f"o{mode}", processing_version="p", metadata=metadata
|
||||
)
|
||||
item = unique_receipt(intake.config.spool, "symlink", b"link")
|
||||
path = intake.config.spool / str(item["spool_name"])
|
||||
path.unlink()
|
||||
path.symlink_to(tmp_path / "outside")
|
||||
assert not intake.accept_helper_receipt(
|
||||
item, occurrence_id="symlink", processing_version="p", metadata=metadata
|
||||
)
|
||||
|
||||
|
||||
def test_open_verified_and_acceptance_close_descriptors(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
intake.armed = True
|
||||
item = unique_receipt(intake.config.spool, "fd", b"reader")
|
||||
before = len(list(Path("/proc/self/fd").iterdir()))
|
||||
assert intake.accept_helper_receipt(
|
||||
item,
|
||||
occurrence_id="fd",
|
||||
processing_version="p",
|
||||
metadata=FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
accepted = intake.take()
|
||||
assert accepted is not None and len(list(Path("/proc/self/fd").iterdir())) == before
|
||||
with intake.open_verified(accepted) as reader:
|
||||
assert b"".join(reader) == PNG_SIGNATURE + b"reader"
|
||||
path = accepted.spool_path
|
||||
path.write_bytes(b"changed")
|
||||
with pytest.raises(ValueError), intake.open_verified(accepted):
|
||||
pass
|
||||
|
||||
|
||||
def test_duplicate_cache_eviction_shutdown_and_unreferenced_candidates(tmp_path: Path) -> None:
|
||||
intake = ClipboardIntake(config(tmp_path, queue=2))
|
||||
intake.armed = True
|
||||
metadata = FocusedWindowMetadata(None, None, None)
|
||||
items = [unique_receipt(intake.config.spool, f"t{i}", bytes([i])) for i in range(3)]
|
||||
for index, item in enumerate(items):
|
||||
assert intake.accept_helper_receipt(
|
||||
item, occurrence_id=f"o{index}", processing_version="p", metadata=metadata
|
||||
)
|
||||
intake.take()
|
||||
assert len(intake._seen_tokens) == 2 and not intake.accept_helper_receipt(
|
||||
items[2], occurrence_id="latest", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert intake.accept_helper_receipt(
|
||||
items[0], occurrence_id="evicted", processing_version="p", metadata=metadata
|
||||
)
|
||||
intake = ClipboardIntake(config(tmp_path / "other", queue=1))
|
||||
intake.armed = True
|
||||
queued = unique_receipt(intake.config.spool, "queued", b"queued")
|
||||
extra = unique_receipt(intake.config.spool, "extra", b"extra")
|
||||
assert intake.accept_helper_receipt(
|
||||
queued, occurrence_id="queued", processing_version="p", metadata=metadata
|
||||
)
|
||||
assert [path.name for path in intake.unreferenced_spool_candidates()] == [
|
||||
str(extra["spool_name"])
|
||||
]
|
||||
intake._helper_payloads.append({"receipt": "real"})
|
||||
intake.shutdown()
|
||||
assert (
|
||||
not intake._queue
|
||||
and not intake._helper_payloads
|
||||
and not intake._seen_tokens
|
||||
and not intake._seen_order
|
||||
)
|
||||
|
||||
|
||||
def test_receipt_reader_bounds_spawn_failures_and_config_types(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
class Output:
|
||||
def __init__(self) -> None:
|
||||
self.parts = [
|
||||
b"x" * 1025 + b"\n",
|
||||
json.dumps({"ok": 1}).encode() + b"\n",
|
||||
json.dumps({"two": 2}).encode() + b"\n",
|
||||
]
|
||||
|
||||
def readline(self, _size: int) -> bytes:
|
||||
return self.parts.pop(0) if self.parts else b""
|
||||
|
||||
class Process:
|
||||
stdout = Output()
|
||||
|
||||
intake = ClipboardIntake(config(tmp_path, queue=1))
|
||||
intake._read_receipts(Process()) # type: ignore[arg-type]
|
||||
assert intake.helper_queue_depth == 1 and intake.take_helper_receipt() == {"ok": 1}
|
||||
attempts = 0
|
||||
|
||||
def failing(*_args: object, **_kwargs: object) -> object:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
raise OSError("missing")
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", failing)
|
||||
failed = ClipboardIntake(config(tmp_path, retries=2))
|
||||
failed.arm()
|
||||
failed.tick(failed._restart_after + 0.01)
|
||||
failed.tick(failed._restart_after + 0.01)
|
||||
failed.tick(failed._restart_after + 1)
|
||||
assert attempts == 3
|
||||
for field, value in (
|
||||
("max_image_bytes", 1.5),
|
||||
("max_queue_items", "1"),
|
||||
("max_restarts", 1.5),
|
||||
("shutdown_timeout_seconds", 0),
|
||||
("restart_backoff_seconds", math.inf),
|
||||
):
|
||||
values: dict[str, object] = dict(
|
||||
spool=(tmp_path / "x").resolve(),
|
||||
max_image_bytes=100,
|
||||
max_queue_items=1,
|
||||
policy_version="v1",
|
||||
max_restarts=1,
|
||||
)
|
||||
values[field] = value
|
||||
with pytest.raises(ValueError):
|
||||
WatchConfig(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_bounded_niri_runner_and_valid_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert (
|
||||
clipboard._bounded_command_output([sys.executable, "-c", "print('x'*20000)"], {}, 1) is None
|
||||
)
|
||||
assert (
|
||||
clipboard._bounded_command_output(
|
||||
[sys.executable, "-c", "import time; time.sleep(2)"], {}, 0.01
|
||||
)
|
||||
is None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
clipboard,
|
||||
"_bounded_command_output",
|
||||
lambda *_args: b'{"app_id":"game","title":"title","id":1}',
|
||||
)
|
||||
assert focused_window() == FocusedWindowMetadata("game", "title", 1)
|
||||
|
||||
|
||||
def test_exited_watcher_is_waited_stdout_closed_and_reader_retired(tmp_path: Path) -> None:
|
||||
class Output:
|
||||
closed = False
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
class Child:
|
||||
pid = os.getpid()
|
||||
stdout = Output()
|
||||
waits = 0
|
||||
|
||||
def poll(self) -> int:
|
||||
return 1
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self.waits += 1
|
||||
return 1
|
||||
|
||||
class Reader:
|
||||
joined = False
|
||||
|
||||
def join(self, timeout: float | None = None) -> None:
|
||||
self.joined = True
|
||||
|
||||
intake = ClipboardIntake(config(tmp_path))
|
||||
child = Child()
|
||||
reader = Reader()
|
||||
intake._watcher = child # type: ignore[assignment]
|
||||
intake._reader = reader # type: ignore[assignment]
|
||||
intake.pause()
|
||||
assert child.waits == 1 and child.stdout.closed and reader.joined and intake._reader is None
|
||||
@@ -0,0 +1,389 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import game_data_capture.ingest.spool_helper as spool_helper
|
||||
from game_data_capture.ingest.spool_helper import (
|
||||
PNG_SIGNATURE,
|
||||
SpoolError,
|
||||
acknowledge_receipt,
|
||||
durable_receipts,
|
||||
receipt_name,
|
||||
reconcile_spool,
|
||||
spool_png,
|
||||
)
|
||||
|
||||
|
||||
def png(payload: bytes = b"data") -> bytes:
|
||||
return PNG_SIGNATURE + payload
|
||||
|
||||
|
||||
def fill_receipt_capacity(spool: Path) -> dict[str, object]:
|
||||
seed = spool_png(
|
||||
io.BytesIO(png(b"seed")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="seed",
|
||||
policy_version="v1",
|
||||
)
|
||||
for index in range(255):
|
||||
token = f"pre{index}"
|
||||
sidecar = spool / receipt_name(token)
|
||||
sidecar.write_text(
|
||||
json.dumps(dict(seed, receipt_token=token), sort_keys=True, separators=(",", ":"))
|
||||
)
|
||||
sidecar.chmod(0o600)
|
||||
return seed
|
||||
|
||||
|
||||
def test_png_spool_is_private_atomic_and_deduplicated(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
replaced: list[tuple[Path, Path]] = []
|
||||
real_replace = os.replace
|
||||
|
||||
def replace(source: Path, destination: Path) -> None:
|
||||
replaced.append((source, destination))
|
||||
real_replace(source, destination)
|
||||
|
||||
monkeypatch.setattr(os, "replace", replace)
|
||||
first = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
second = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="two", policy_version="v1"
|
||||
)
|
||||
assert first["sha256"] == second["sha256"]
|
||||
assert (spool.stat().st_mode & 0o777) == 0o700
|
||||
installed = spool / str(first["spool_name"])
|
||||
assert installed.read_bytes() == png() and (installed.stat().st_mode & 0o777) == 0o600
|
||||
assert len(replaced) == 1 and replaced[0][1] == installed
|
||||
sidecars = sorted(spool.glob("*.receipt.json"))
|
||||
assert [sidecar.name for sidecar in sidecars] == ["one.receipt.json", "two.receipt.json"]
|
||||
assert all((sidecar.stat().st_mode & 0o777) == 0o600 for sidecar in sidecars)
|
||||
assert all(set(json.loads(sidecar.read_text())) == set(first) for sidecar in sidecars)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [b"", b"text", PNG_SIGNATURE[:4]])
|
||||
def test_empty_text_and_malformed_png_leave_no_partial(tmp_path: Path, value: bytes) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(value), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
def test_unsupported_mime_leaves_no_partial(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
media_type="text/plain",
|
||||
)
|
||||
assert not spool.exists()
|
||||
|
||||
|
||||
def test_short_reads_accumulate_the_png_signature(tmp_path: Path) -> None:
|
||||
class ShortReads:
|
||||
def __init__(self) -> None:
|
||||
self.parts = [PNG_SIGNATURE[:3], PNG_SIGNATURE[3:] + b"payload", b""]
|
||||
|
||||
def read(self, _size: int) -> bytes:
|
||||
return self.parts.pop(0)
|
||||
|
||||
receipt = spool_png(
|
||||
ShortReads(),
|
||||
spool=(tmp_path / "spool").resolve(),
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert receipt["byte_size"] == len(PNG_SIGNATURE + b"payload")
|
||||
|
||||
|
||||
def test_size_bound_and_crash_reconciliation_remove_partials(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
with pytest.raises(SpoolError):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"x" * 100)),
|
||||
spool=spool,
|
||||
max_bytes=10,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
leftover = spool / ".crash.partial"
|
||||
leftover.write_bytes(b"incomplete")
|
||||
assert reconcile_spool(spool) == [".crash.partial"]
|
||||
assert not leftover.exists()
|
||||
|
||||
|
||||
def test_pending_receipt_is_promoted_only_when_its_png_is_verified(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
receipt = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
final = spool / receipt_name("one")
|
||||
pending = spool / ".one.receipt.pending"
|
||||
os.replace(final, pending)
|
||||
assert reconcile_spool(spool) == []
|
||||
assert final.exists() and not pending.exists()
|
||||
final.unlink()
|
||||
pending.write_text(json.dumps(receipt))
|
||||
assert reconcile_spool(spool) == [pending.name]
|
||||
assert not pending.exists()
|
||||
|
||||
|
||||
def test_receipt_token_replay_cannot_replace_the_original_sidecar(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
original = spool_png(
|
||||
io.BytesIO(png(b"original")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
final = spool / receipt_name("same-token")
|
||||
original_bytes = final.read_bytes()
|
||||
assert (
|
||||
spool_png(
|
||||
io.BytesIO(png(b"original")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
== original
|
||||
)
|
||||
with pytest.raises(SpoolError, match="receipt token conflict"):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"different")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="same-token",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert final.read_bytes() == original_bytes
|
||||
conflicting = dict(original, sha256="0" * 64, spool_name=f"{'0' * 64}.png")
|
||||
pending = spool / ".same-token.receipt.pending"
|
||||
pending.write_text(json.dumps(conflicting, sort_keys=True, separators=(",", ":")))
|
||||
pending.chmod(0o600)
|
||||
assert pending.name in reconcile_spool(spool)
|
||||
assert final.read_bytes() == original_bytes and not pending.exists()
|
||||
|
||||
|
||||
def test_sidecars_are_bounded_validated_and_acknowledged_exactly_once(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
first = spool_png(
|
||||
io.BytesIO(png(b"one")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="one",
|
||||
policy_version="v1",
|
||||
)
|
||||
second = spool_png(
|
||||
io.BytesIO(png(b"two")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="two",
|
||||
policy_version="v1",
|
||||
)
|
||||
malformed = spool / "bad.receipt.json"
|
||||
malformed.write_bytes(b"{" + b"x" * 1024)
|
||||
malformed.chmod(0o600)
|
||||
linked = spool / "linked.receipt.json"
|
||||
linked.symlink_to(malformed)
|
||||
assert {item["receipt_token"] for item in durable_receipts(spool, max_items=2)} == {
|
||||
"one",
|
||||
"two",
|
||||
}
|
||||
assert acknowledge_receipt(spool, first)
|
||||
assert not acknowledge_receipt(spool, first)
|
||||
assert (spool / receipt_name("two")).exists()
|
||||
assert not acknowledge_receipt(spool, dict(second, receipt_token="one"))
|
||||
|
||||
|
||||
def test_malformed_sidecars_do_not_starve_valid_receipts_or_partial_cleanup(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
for index in range(256):
|
||||
malformed = spool / f"{index:03d}.receipt.json"
|
||||
malformed.write_bytes(b"{")
|
||||
malformed.chmod(0o600)
|
||||
valid = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="valid", policy_version="v1"
|
||||
)
|
||||
assert list(durable_receipts(spool, max_items=1)) == [valid]
|
||||
assert list(spool.glob("*.receipt.json")) == [spool / "valid.receipt.json"]
|
||||
for index in range(256):
|
||||
pending = spool / f".pending{index}.receipt.pending"
|
||||
pending.write_text(
|
||||
json.dumps(
|
||||
dict(valid, receipt_token=f"pending{index}"), sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
)
|
||||
pending.chmod(0o600)
|
||||
partial = spool / ".after-pending.partial"
|
||||
partial.write_bytes(b"incomplete")
|
||||
assert partial.name in reconcile_spool(spool)
|
||||
assert not partial.exists()
|
||||
|
||||
|
||||
def test_receipt_capacity_rejects_new_tokens_but_accepts_exact_replay(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
seed = fill_receipt_capacity(spool)
|
||||
evidence = (spool / receipt_name("seed")).read_bytes()
|
||||
png_names = {path.name for path in spool.glob("*.png")}
|
||||
with pytest.raises(SpoolError, match="capacity"):
|
||||
spool_png(
|
||||
io.BytesIO(png(b"new")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="new",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert (spool / receipt_name("seed")).read_bytes() == evidence
|
||||
assert {path.name for path in spool.glob("*.png")} == png_names
|
||||
assert (
|
||||
spool_png(
|
||||
io.BytesIO(png(b"seed")),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="seed",
|
||||
policy_version="v1",
|
||||
)
|
||||
== seed
|
||||
)
|
||||
|
||||
|
||||
def test_receipt_capacity_reconciles_malformed_files_before_counting(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
for index in range(256):
|
||||
malformed = spool / f"bad{index}.receipt.json"
|
||||
malformed.write_bytes(b"{")
|
||||
malformed.chmod(0o600)
|
||||
accepted = spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="accepted", policy_version="v1"
|
||||
)
|
||||
assert accepted["receipt_token"] == "accepted"
|
||||
assert list(spool.glob("bad*.receipt.json")) == []
|
||||
|
||||
|
||||
def test_receipt_capacity_is_serialized_across_helper_processes(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
fill_receipt_capacity(spool)
|
||||
for index in range(1):
|
||||
(spool / receipt_name(f"pre{index}")).unlink()
|
||||
arguments = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"game_data_capture.ingest.spool_helper",
|
||||
"--spool",
|
||||
str(spool),
|
||||
"--max-bytes",
|
||||
"100",
|
||||
"--policy-version",
|
||||
"v1",
|
||||
]
|
||||
first = subprocess.Popen([*arguments, "--receipt-token", "race-one"], stdin=subprocess.PIPE)
|
||||
second = subprocess.Popen([*arguments, "--receipt-token", "race-two"], stdin=subprocess.PIPE)
|
||||
assert first.stdin is not None and second.stdin is not None
|
||||
first.stdin.write(png(b"one"))
|
||||
first.stdin.close()
|
||||
second.stdin.write(png(b"two"))
|
||||
second.stdin.close()
|
||||
assert sorted((first.wait(), second.wait())) == [0, 2]
|
||||
assert len(list(spool.glob("*.receipt.json"))) == 256
|
||||
|
||||
|
||||
def test_spool_lock_is_private_rejects_links_and_releases_on_failure(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
spool.mkdir(mode=0o700)
|
||||
(spool / ".spool.lock").symlink_to(tmp_path / "outside")
|
||||
with pytest.raises(SpoolError, match="unsafe spool lock"):
|
||||
spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="link", policy_version="v1"
|
||||
)
|
||||
(spool / ".spool.lock").unlink()
|
||||
with monkeypatch.context() as context:
|
||||
context.setattr(
|
||||
spool_helper,
|
||||
"_install_pending_sidecar",
|
||||
lambda *_args: (_ for _ in ()).throw(OSError()),
|
||||
)
|
||||
with pytest.raises(OSError):
|
||||
spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="failure",
|
||||
policy_version="v1",
|
||||
)
|
||||
accepted = spool_png(
|
||||
io.BytesIO(png()),
|
||||
spool=spool,
|
||||
max_bytes=100,
|
||||
receipt_token="recovered",
|
||||
policy_version="v1",
|
||||
)
|
||||
assert accepted["receipt_token"] == "recovered"
|
||||
assert (spool / ".spool.lock").stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_short_write_removes_partial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
monkeypatch.setattr(os, "write", lambda _fd, _data: 0)
|
||||
with pytest.raises(OSError):
|
||||
spool_png(
|
||||
io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1"
|
||||
)
|
||||
assert not list(spool.glob("*.partial"))
|
||||
|
||||
|
||||
def test_helper_source_has_no_service_or_storage_imports() -> None:
|
||||
source = Path(__file__).parents[3] / "src/game_data_capture/ingest/spool_helper.py"
|
||||
text = source.read_text(encoding="utf-8")
|
||||
for forbidden in ("fastapi", "sqlite", "openai", "game_data_capture.storage"):
|
||||
assert forbidden not in text.lower()
|
||||
|
||||
|
||||
def test_stdin_entrypoint_returns_only_bounded_json_receipt(tmp_path: Path) -> None:
|
||||
spool = (tmp_path / "spool").resolve()
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"game_data_capture.ingest.spool_helper",
|
||||
"--spool",
|
||||
str(spool),
|
||||
"--max-bytes",
|
||||
"100",
|
||||
"--receipt-token",
|
||||
"receipt",
|
||||
"--policy-version",
|
||||
"v1",
|
||||
],
|
||||
input=png(),
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env={"PATH": os.environ.get("PATH", "")},
|
||||
)
|
||||
assert completed.returncode == 0 and len(completed.stdout) <= 1024 and not completed.stderr
|
||||
receipt = json.loads(completed.stdout)
|
||||
assert receipt["media_type"] == "image/png" and receipt["receipt_token"] == "receipt"
|
||||
@@ -0,0 +1,466 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from contextlib import AbstractContextManager
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import uvicorn
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
import game_data_capture.runtime as runtime_module
|
||||
from game_data_capture.config import AppConfig, CaptureLimits, LifecycleConfig, ServerConfig
|
||||
from game_data_capture.ingest.clipboard import IntakeReceipt
|
||||
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE
|
||||
from game_data_capture.paths import RuntimePaths
|
||||
from game_data_capture.runtime import (
|
||||
CaptureRuntime,
|
||||
InstanceLock,
|
||||
IntakePort,
|
||||
RuntimeFailure,
|
||||
_CaptureControl,
|
||||
)
|
||||
from game_data_capture.storage import CaptureCorpus, CaptureRepository, StorageDatabase
|
||||
|
||||
_PNG = PNG_SIGNATURE + b"unit-png"
|
||||
|
||||
|
||||
def _paths(tmp_path: Path) -> RuntimePaths:
|
||||
root = tmp_path / "xdg"
|
||||
return RuntimePaths(
|
||||
config_dir=(root / "config").resolve(),
|
||||
data_dir=(root / "data").resolve(),
|
||||
cache_dir=(root / "cache").resolve(),
|
||||
runtime_dir=(root / "runtime").resolve(),
|
||||
)
|
||||
|
||||
|
||||
def _runtime(tmp_path: Path, *, limits: CaptureLimits | None = None) -> CaptureRuntime:
|
||||
return CaptureRuntime(
|
||||
AppConfig(
|
||||
lifecycle=LifecycleConfig(lock_timeout_seconds=0), limits=limits or CaptureLimits()
|
||||
),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
)
|
||||
|
||||
|
||||
def test_one_shot_persists_distinct_occurrences_and_one_blob(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path)
|
||||
image = tmp_path / "image.png"
|
||||
image.write_bytes(_PNG)
|
||||
|
||||
first = runtime.ingest_image(image)
|
||||
second = runtime.ingest_image(image)
|
||||
database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir)
|
||||
|
||||
with database.connect() as connection:
|
||||
assert first != second
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 2
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["malformed", "oversize", "symlink", "broad-mode"])
|
||||
def test_one_shot_rejects_unsafe_png_inputs(tmp_path: Path, kind: str) -> None:
|
||||
runtime = _runtime(tmp_path, limits=CaptureLimits(max_image_bytes=len(_PNG)))
|
||||
image = tmp_path / "image.png"
|
||||
image.write_bytes(_PNG)
|
||||
if kind == "malformed":
|
||||
image.write_bytes(b"not-a-png")
|
||||
elif kind == "oversize":
|
||||
image.write_bytes(_PNG + b"x")
|
||||
elif kind == "symlink":
|
||||
linked = tmp_path / "linked.png"
|
||||
linked.symlink_to(image)
|
||||
image = linked
|
||||
else:
|
||||
image.chmod(0o666)
|
||||
try:
|
||||
with pytest.raises(RuntimeFailure, match="safe PNG"):
|
||||
runtime.ingest_image(image)
|
||||
finally:
|
||||
if image.exists() and not image.is_symlink():
|
||||
image.chmod(0o600)
|
||||
|
||||
|
||||
def test_stale_lock_file_is_reused_but_an_active_lock_is_not(tmp_path: Path) -> None:
|
||||
paths = _paths(tmp_path)
|
||||
paths.runtime_dir.mkdir(parents=True)
|
||||
paths.instance_lock_file.write_text("stale", encoding="utf-8")
|
||||
paths.instance_lock_file.chmod(0o600)
|
||||
first = InstanceLock.acquire(paths.instance_lock_file, 0)
|
||||
try:
|
||||
with pytest.raises(RuntimeFailure, match="another capture instance is active"):
|
||||
InstanceLock.acquire(paths.instance_lock_file, 0)
|
||||
finally:
|
||||
first.release()
|
||||
second = InstanceLock.acquire(paths.instance_lock_file, 0)
|
||||
second.release()
|
||||
assert paths.instance_lock_file.exists()
|
||||
|
||||
|
||||
class InterruptingIntake:
|
||||
def __init__(self) -> None:
|
||||
self.shutdown_called = False
|
||||
|
||||
def reconcile(self) -> list[str]:
|
||||
return []
|
||||
|
||||
def arm(self) -> None:
|
||||
return None
|
||||
|
||||
def tick(self) -> None:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
def take_helper_receipt(self) -> dict[str, object] | None:
|
||||
return None
|
||||
|
||||
def accept_helper_receipt(
|
||||
self, payload: Mapping[str, object], *, occurrence_id: str, processing_version: str
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def take(self) -> IntakeReceipt | None:
|
||||
return None
|
||||
|
||||
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]:
|
||||
raise AssertionError("no receipt should be opened")
|
||||
|
||||
def acknowledge(self, receipt: IntakeReceipt) -> bool:
|
||||
raise AssertionError("no receipt should be acknowledged")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.shutdown_called = True
|
||||
|
||||
|
||||
def test_keyboard_interrupt_reaps_intake_and_releases_runtime_lock(tmp_path: Path) -> None:
|
||||
intake = InterruptingIntake()
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
runtime.watch(poll_seconds=0.001)
|
||||
|
||||
assert intake.shutdown_called
|
||||
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
reusable.release()
|
||||
|
||||
|
||||
def test_cancelled_watch_cleans_up_without_entering_the_tick_loop(tmp_path: Path) -> None:
|
||||
cancelled = Event()
|
||||
cancelled.set()
|
||||
intake = InterruptingIntake()
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
assert intake.shutdown_called
|
||||
|
||||
|
||||
def test_one_shot_never_connects_outside_the_utility_data_root(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
runtime = _runtime(tmp_path)
|
||||
image = tmp_path / "image.png"
|
||||
image.write_bytes(_PNG)
|
||||
real_connect = cast(Callable[..., sqlite3.Connection], sqlite3.connect)
|
||||
connected: list[Path] = []
|
||||
|
||||
def checked_connect(database: str | Path, **kwargs: object) -> sqlite3.Connection:
|
||||
connected.append(Path(database))
|
||||
return real_connect(database, **kwargs)
|
||||
|
||||
monkeypatch.setattr(sqlite3, "connect", checked_connect)
|
||||
runtime.ingest_image(image)
|
||||
assert connected
|
||||
assert all(path.parent == runtime.paths.data_dir for path in connected)
|
||||
|
||||
|
||||
class ControlledIntake:
|
||||
def __init__(self, *, fail_tick: bool = False) -> None:
|
||||
self.armed = False
|
||||
self.reconciled = False
|
||||
self.shutdown_called = Event()
|
||||
self.queue_depth = 0
|
||||
self.helper_queue_depth = 0
|
||||
self.fail_tick = fail_tick
|
||||
|
||||
def reconcile(self) -> list[str]:
|
||||
self.reconciled = True
|
||||
return []
|
||||
|
||||
def arm(self) -> None:
|
||||
self.armed = True
|
||||
|
||||
def tick(self) -> None:
|
||||
if self.fail_tick:
|
||||
raise OSError("helper fault")
|
||||
|
||||
def take_helper_receipt(self) -> dict[str, object] | None:
|
||||
return None
|
||||
|
||||
def accept_helper_receipt(
|
||||
self, payload: Mapping[str, object], *, occurrence_id: str, processing_version: str
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def take(self) -> IntakeReceipt | None:
|
||||
return None
|
||||
|
||||
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]:
|
||||
raise AssertionError("no receipt should be opened")
|
||||
|
||||
def acknowledge(self, receipt: IntakeReceipt) -> bool:
|
||||
raise AssertionError("no receipt should be acknowledged")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.shutdown_called.set()
|
||||
|
||||
|
||||
class FaultingCleanupIntake(ControlledIntake):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(fail_tick=True)
|
||||
self.cleanup_started = Event()
|
||||
self.cleanup_release = Event()
|
||||
self._shutdown_calls = 0
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutdown_calls += 1
|
||||
self.cleanup_started.set()
|
||||
if self._shutdown_calls == 1:
|
||||
self.cleanup_release.wait()
|
||||
self.shutdown_called.set()
|
||||
|
||||
|
||||
class MalformedQueueIntake:
|
||||
@property
|
||||
def queue_depth(self) -> object:
|
||||
raise OSError("private intake failure")
|
||||
|
||||
@property
|
||||
def helper_queue_depth(self) -> object:
|
||||
return True
|
||||
|
||||
|
||||
class EmptyRepository:
|
||||
def receive(
|
||||
self,
|
||||
capture_id: str,
|
||||
receipt_token: str,
|
||||
chunks: Iterable[bytes],
|
||||
*,
|
||||
media_type: str,
|
||||
max_bytes: int,
|
||||
) -> str:
|
||||
raise AssertionError("no receipt should be stored")
|
||||
|
||||
|
||||
def test_control_serializes_repeated_start_stop_and_reports_bounded_queue(tmp_path: Path) -> None:
|
||||
intake = ControlledIntake()
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository()))
|
||||
outcomes: list[str] = []
|
||||
|
||||
def start() -> None:
|
||||
outcomes.append(control.start_session())
|
||||
|
||||
first, second = Thread(target=start), Thread(target=start)
|
||||
first.start()
|
||||
second.start()
|
||||
first.join()
|
||||
second.join()
|
||||
|
||||
assert sorted(outcomes) == ["already_armed", "started"]
|
||||
assert intake.reconciled and intake.armed
|
||||
assert control.status().queue_depth == 0
|
||||
assert control.status().queue_capacity == runtime.config.limits.max_queue_items
|
||||
assert control.stop_session() in {"stopped", "already_stopped"}
|
||||
assert control.stop_session() == "already_stopped"
|
||||
assert intake.shutdown_called.wait(0.1)
|
||||
|
||||
|
||||
def test_control_status_combines_and_bounds_both_intake_queues(tmp_path: Path) -> None:
|
||||
intake = ControlledIntake()
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository()))
|
||||
|
||||
intake.queue_depth, intake.helper_queue_depth = 2, 3
|
||||
control._intake = cast(IntakePort, intake)
|
||||
assert control.status().queue_depth == 5
|
||||
intake.queue_depth, intake.helper_queue_depth = 1000, -1
|
||||
assert control.status().queue_depth == runtime.config.limits.max_queue_items
|
||||
control._intake = cast(IntakePort, MalformedQueueIntake())
|
||||
assert control.status().queue_depth == 0
|
||||
|
||||
|
||||
def test_control_turns_background_intake_fault_into_bounded_disarmed_state(tmp_path: Path) -> None:
|
||||
intake = ControlledIntake(fail_tick=True)
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository()))
|
||||
|
||||
assert control.start_session() == "started"
|
||||
assert intake.shutdown_called.wait(0.2)
|
||||
assert control.stop_session() in {"stopped", "already_stopped"}
|
||||
assert control.status().armed is False
|
||||
assert control.status().queue_depth == 0
|
||||
|
||||
|
||||
def test_service_retains_its_lock_until_faulting_worker_cleanup_finishes(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
intake = FaultingCleanupIntake()
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0, shutdown_timeout_seconds=1)),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
captured: dict[str, _CaptureControl] = {}
|
||||
|
||||
def factory(**kwargs: object) -> object:
|
||||
captured["control"] = cast(_CaptureControl, kwargs["capture_control"])
|
||||
return object()
|
||||
|
||||
def runner(_: object, **__: object) -> None:
|
||||
assert captured["control"].start_session() == "started"
|
||||
assert intake.cleanup_started.wait(0.2)
|
||||
|
||||
monkeypatch.setattr(runtime_module, "create_app", factory)
|
||||
monkeypatch.setattr(uvicorn, "run", runner)
|
||||
failure: list[RuntimeFailure] = []
|
||||
|
||||
def serve() -> None:
|
||||
try:
|
||||
runtime.serve()
|
||||
except RuntimeFailure as error:
|
||||
failure.append(error)
|
||||
|
||||
service = Thread(target=serve)
|
||||
service.start()
|
||||
try:
|
||||
service.join(timeout=2)
|
||||
assert not service.is_alive()
|
||||
assert failure
|
||||
with pytest.raises(RuntimeFailure, match="another capture instance"):
|
||||
InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
outcomes: list[str] = []
|
||||
start = Thread(target=lambda: outcomes.append(captured["control"].start_session()))
|
||||
stop = Thread(target=lambda: outcomes.append(captured["control"].stop_session()))
|
||||
start.start()
|
||||
stop.start()
|
||||
start.join()
|
||||
stop.join()
|
||||
assert sorted(outcomes) == ["conflict", "conflict"]
|
||||
finally:
|
||||
intake.cleanup_release.set()
|
||||
for _ in range(50):
|
||||
try:
|
||||
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
except RuntimeFailure:
|
||||
Event().wait(0.01)
|
||||
else:
|
||||
reusable.release()
|
||||
break
|
||||
else:
|
||||
pytest.fail("service lock was not released after worker cleanup")
|
||||
|
||||
|
||||
def test_serve_constructs_durable_corpus_without_auto_arming_and_uses_one_fragment_token(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
created: list[ControlledIntake] = []
|
||||
token = "A" * 43
|
||||
|
||||
def factory(_: object) -> ControlledIntake:
|
||||
intake = ControlledIntake()
|
||||
created.append(intake)
|
||||
return intake
|
||||
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(
|
||||
server=ServerConfig(host="127.0.0.1", port=9876),
|
||||
lifecycle=LifecycleConfig(lock_timeout_seconds=0),
|
||||
),
|
||||
_paths(tmp_path),
|
||||
migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
intake_factory=factory,
|
||||
)
|
||||
monkeypatch.setattr(runtime_module, "new_bootstrap_token", lambda: token)
|
||||
received: dict[str, object] = {}
|
||||
|
||||
def runner(app: object, **kwargs: object) -> None:
|
||||
received.update(kwargs)
|
||||
with TestClient(cast(ASGIApp, app), base_url="http://127.0.0.1:9876") as client:
|
||||
assert client.get("/health").json() == {"status": "ok"}
|
||||
assert client.get("/").status_code == 200
|
||||
assert created == []
|
||||
bootstrapped = client.post(
|
||||
"/api/bootstrap",
|
||||
json={"token": token},
|
||||
headers={"Origin": "http://127.0.0.1:9876"},
|
||||
)
|
||||
csrf = bootstrapped.json()["csrf_token"]
|
||||
assert client.get("/api/session").json()["armed"] is False
|
||||
started = client.post(
|
||||
"/api/session/start",
|
||||
json={"confirmation": "START"},
|
||||
headers={"Origin": "http://127.0.0.1:9876", "X-CSRF-Token": csrf},
|
||||
)
|
||||
assert started.json()["result"] == "started"
|
||||
assert created[0].reconciled and created[0].armed
|
||||
|
||||
monkeypatch.setattr(uvicorn, "run", runner)
|
||||
runtime.serve()
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert output.count(token) == 1
|
||||
assert f"http://127.0.0.1:9876/#bootstrap={token}" in output
|
||||
assert received == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 9876,
|
||||
"reload": False,
|
||||
"workers": 1,
|
||||
"proxy_headers": False,
|
||||
}
|
||||
assert created[0].shutdown_called.wait(0.1)
|
||||
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
reusable.release()
|
||||
|
||||
|
||||
def test_service_startup_reconciliation_failure_releases_the_instance_lock(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
runtime = _runtime(tmp_path)
|
||||
|
||||
def failed_reconciliation(self: CaptureCorpus, *, limit: int = 64) -> int:
|
||||
raise OSError("private deletion recovery fault")
|
||||
|
||||
monkeypatch.setattr(CaptureCorpus, "reconcile_deletions", failed_reconciliation)
|
||||
with pytest.raises(RuntimeFailure, match="initialization failed"):
|
||||
runtime.serve()
|
||||
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
reusable.release()
|
||||
@@ -0,0 +1,200 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.storage.blobs import (
|
||||
BlobCorruptionError,
|
||||
BlobLimitError,
|
||||
BlobLockError,
|
||||
ContentAddressedBlobStore,
|
||||
)
|
||||
|
||||
|
||||
def test_blob_is_verified_deduplicated_and_private(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
payload = b"capture-bytes"
|
||||
digest, size = store.put([payload], max_bytes=100)
|
||||
assert digest == hashlib.sha256(payload).hexdigest()
|
||||
assert store.put([payload], max_bytes=100) == (digest, size)
|
||||
assert b"".join(store.read(digest, max_bytes=100)) == payload
|
||||
assert (store._path(digest).stat().st_mode & 0o777) == 0o600
|
||||
|
||||
|
||||
def test_corruption_and_partial_files_are_detected_or_reconciled(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
digest, _ = store.put([b"good"], max_bytes=100)
|
||||
store._path(digest).write_bytes(b"bad")
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
store.verify(digest)
|
||||
store.temporary.mkdir(parents=True, exist_ok=True)
|
||||
partial = store.temporary / ".crash.partial"
|
||||
partial.write_bytes(b"partial")
|
||||
assert store.reconcile() == [".crash.partial"]
|
||||
assert not partial.exists()
|
||||
|
||||
|
||||
def test_rejects_hash_and_generator_failures_without_partial_file(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
store.put([b"value"], max_bytes=100, expected_sha256="0" * 64)
|
||||
|
||||
def broken() -> Iterator[bytes]:
|
||||
yield b"prefix"
|
||||
raise RuntimeError("source failed")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
store.put(broken(), max_bytes=100)
|
||||
assert not list(store.temporary.glob("*.partial"))
|
||||
|
||||
|
||||
def test_root_symlinks_are_not_traversed(tmp_path: Path) -> None:
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
root = (tmp_path / "data").resolve()
|
||||
root.symlink_to(outside, target_is_directory=True)
|
||||
store = ContentAddressedBlobStore(root)
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
store.put([b"x"], max_bytes=1)
|
||||
|
||||
|
||||
def test_limits_and_root_symlink_reconciliation_are_safe(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
for limit in (0, -1, True):
|
||||
with pytest.raises(BlobLimitError):
|
||||
store.put([b"x"], max_bytes=limit)
|
||||
digest, _ = store.put([b"x"], max_bytes=1)
|
||||
for limit in (0, -1, True):
|
||||
with pytest.raises(BlobLimitError):
|
||||
list(store.read(digest, max_bytes=limit))
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
partial = outside / ".outside.partial"
|
||||
partial.write_bytes(b"outside")
|
||||
store.temporary.rmdir()
|
||||
store.temporary.symlink_to(outside, target_is_directory=True)
|
||||
assert store.reconcile() == []
|
||||
assert partial.read_bytes() == b"outside"
|
||||
|
||||
|
||||
def test_corrupt_read_rejects_before_yielding(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
digest, _ = store.put([b"good"], max_bytes=10)
|
||||
store._path(digest).write_bytes(b"bad")
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
list(store.read(digest, max_bytes=10))
|
||||
|
||||
|
||||
def test_oversize_and_valid_digest_mismatch_leave_no_partial(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
with pytest.raises(BlobLimitError):
|
||||
store.put([b"too-large"], max_bytes=1)
|
||||
assert not list(store.temporary.glob("*.partial"))
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
store.put([b"value"], max_bytes=100, expected_sha256="0" * 64)
|
||||
assert not list(store.temporary.glob("*.partial"))
|
||||
|
||||
|
||||
def test_blob_root_symlink_purge_does_not_inspect_outside(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
store.put([b"inside"], max_bytes=100)
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
outside_file = outside / "outside-blob"
|
||||
outside_file.write_bytes(b"outside")
|
||||
for entry in store.blobs.iterdir():
|
||||
if entry.is_file():
|
||||
entry.unlink()
|
||||
else:
|
||||
for child in entry.iterdir():
|
||||
child.unlink()
|
||||
entry.rmdir()
|
||||
store.blobs.rmdir()
|
||||
store.blobs.symlink_to(outside, target_is_directory=True)
|
||||
assert store.purge_dry_run(set()) == []
|
||||
assert outside_file.read_bytes() == b"outside"
|
||||
|
||||
|
||||
def test_purge_dry_run_does_not_follow_symlink(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
digest, _ = store.put([b"inside"], max_bytes=100)
|
||||
outside = tmp_path / "outside"
|
||||
outside.write_bytes(b"outside")
|
||||
unsafe = store.blobs / "ff"
|
||||
unsafe.symlink_to(tmp_path)
|
||||
assert store.purge_dry_run(set()) == [store._path(digest).relative_to(store.root)]
|
||||
assert outside.read_bytes() == b"outside"
|
||||
|
||||
|
||||
def test_operation_lock_rejects_symlink_and_unsafe_mode(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
store.root.mkdir(mode=0o700)
|
||||
outside = tmp_path / "outside"
|
||||
outside.write_bytes(b"x")
|
||||
store.lock_path.symlink_to(outside)
|
||||
with pytest.raises(BlobLockError), store.operation_lock():
|
||||
pass
|
||||
store.lock_path.unlink()
|
||||
store.lock_path.write_bytes(b"")
|
||||
store.lock_path.chmod(0o644)
|
||||
with pytest.raises(BlobLockError), store.operation_lock():
|
||||
pass
|
||||
|
||||
|
||||
def test_operation_lock_timeout_and_release_after_exception(tmp_path: Path) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
entered = threading.Event()
|
||||
outcome: list[type[Exception]] = []
|
||||
|
||||
def contender() -> None:
|
||||
entered.set()
|
||||
try:
|
||||
with store.operation_lock(timeout_seconds=0.02):
|
||||
pass
|
||||
except BlobLockError as error:
|
||||
outcome.append(type(error))
|
||||
|
||||
with store.operation_lock():
|
||||
thread = threading.Thread(target=contender)
|
||||
thread.start()
|
||||
assert entered.wait(0.2)
|
||||
thread.join(0.5)
|
||||
assert not thread.is_alive()
|
||||
assert outcome == [BlobLockError]
|
||||
with pytest.raises(RuntimeError), store.operation_lock():
|
||||
raise RuntimeError("fault")
|
||||
with store.operation_lock(timeout_seconds=0.02):
|
||||
pass
|
||||
for invalid in (True, "1", 0, 6):
|
||||
with (
|
||||
pytest.raises(ValueError),
|
||||
store.operation_lock(
|
||||
timeout_seconds=invalid # type: ignore[arg-type]
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def test_expected_size_mismatch_rejects_before_blob_read(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
store = ContentAddressedBlobStore((tmp_path / "data").resolve())
|
||||
digest, _ = store.put([b"value"], max_bytes=100)
|
||||
store._path(digest).write_bytes(b"x" * 4096)
|
||||
reads = 0
|
||||
original = os.read
|
||||
|
||||
def forbidden_read(fd: int, size: int) -> bytes:
|
||||
nonlocal reads
|
||||
reads += 1
|
||||
return original(fd, size)
|
||||
|
||||
monkeypatch.setattr(os, "read", forbidden_read)
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
store.verify(digest, expected_size=5)
|
||||
assert reads == 0
|
||||
@@ -0,0 +1,192 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.cli import EXIT_CONFIG_ERROR, EXIT_UNIMPLEMENTED, main
|
||||
from game_data_capture.logging import RedactingFormatter, configure_logging, redact_log_data
|
||||
|
||||
|
||||
def test_doctor_reports_required_diagnostics_without_openai_value(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setenv("HOME", str(tmp_path / "home"))
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-this-must-not-appear")
|
||||
output = io.StringIO()
|
||||
|
||||
with redirect_stdout(output):
|
||||
result = main(["doctor", "--json"])
|
||||
|
||||
report = output.getvalue()
|
||||
assert result == 0
|
||||
for name in ("niri", "wl-paste", "ffmpeg", "gpu_provider", "storage_permissions", "openai"):
|
||||
assert f'"name":"{name}"' in report
|
||||
assert "sk-this-must-not-appear" not in report
|
||||
|
||||
|
||||
def test_serve_rejects_public_binding() -> None:
|
||||
error = io.StringIO()
|
||||
|
||||
with redirect_stderr(error):
|
||||
result = main(["serve", "--host", "0.0.0.0"])
|
||||
|
||||
assert result == EXIT_CONFIG_ERROR
|
||||
assert "loopback IP literal" in error.getvalue()
|
||||
|
||||
|
||||
def test_serve_rejects_invalid_port_without_echoing_value() -> None:
|
||||
error = io.StringIO()
|
||||
|
||||
with redirect_stderr(error), pytest.raises(SystemExit) as raised:
|
||||
main(["serve", "--port", "99999-private-value"])
|
||||
|
||||
assert raised.value.code == EXIT_CONFIG_ERROR
|
||||
assert "port must be an integer from 1024 to 65535" in error.getvalue()
|
||||
assert "99999-private-value" not in error.getvalue()
|
||||
|
||||
|
||||
def test_subsystem_commands_are_explicit_stubs() -> None:
|
||||
error = io.StringIO()
|
||||
|
||||
with redirect_stderr(error):
|
||||
result = main(["watch"])
|
||||
|
||||
assert result == EXIT_UNIMPLEMENTED
|
||||
assert "requires gdc-runtime-integration" in error.getvalue()
|
||||
|
||||
|
||||
def test_unimplemented_command_validates_config_before_returning_stub_status(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config_path = tmp_path / "invalid.toml"
|
||||
config_path.write_text("[server]\nport = 80\n", encoding="utf-8")
|
||||
error = io.StringIO()
|
||||
|
||||
with redirect_stderr(error):
|
||||
result = main(["watch", "--config", str(config_path)])
|
||||
|
||||
assert result == EXIT_CONFIG_ERROR
|
||||
assert "server.port" in error.getvalue()
|
||||
assert "subsystem is not implemented" not in error.getvalue()
|
||||
|
||||
|
||||
def test_logging_redacts_sensitive_fields_and_messages() -> None:
|
||||
redacted = redact_log_data(
|
||||
{
|
||||
"authorization": "Bearer secret-value",
|
||||
"image_payload": "data:image/png;base64,abc",
|
||||
"ocr_text": "unrestricted source text",
|
||||
"status": "ok",
|
||||
}
|
||||
)
|
||||
record = logging.LogRecord(
|
||||
"test", logging.INFO, __file__, 1, "Authorization: Bearer hidden sk-visible123", (), None
|
||||
)
|
||||
|
||||
assert redacted == {
|
||||
"authorization": "[REDACTED]",
|
||||
"image_payload": "[REDACTED]",
|
||||
"ocr_text": "[REDACTED]",
|
||||
"status": "ok",
|
||||
}
|
||||
assert "hidden" not in RedactingFormatter("%(message)s").format(record)
|
||||
assert "sk-visible123" not in RedactingFormatter("%(message)s").format(record)
|
||||
|
||||
|
||||
def test_configure_logging_replaces_unredacted_handlers_and_redacts_exceptions() -> None:
|
||||
package_logger = logging.getLogger("game_data_capture")
|
||||
package_handlers = package_logger.handlers[:]
|
||||
package_level = package_logger.level
|
||||
package_propagate = package_logger.propagate
|
||||
root_logger = logging.getLogger()
|
||||
root_handler = logging.NullHandler()
|
||||
root_logger.addHandler(root_handler)
|
||||
leaked_output = io.StringIO()
|
||||
existing_handler = logging.StreamHandler(leaked_output)
|
||||
package_logger.handlers = [existing_handler]
|
||||
captured_output = io.StringIO()
|
||||
|
||||
try:
|
||||
with redirect_stderr(captured_output):
|
||||
configured_logger = configure_logging()
|
||||
configure_logging()
|
||||
try:
|
||||
raise RuntimeError("Authorization: Bearer exception-secret; ocr_text=private text")
|
||||
except RuntimeError:
|
||||
configured_logger.exception("capture failed")
|
||||
|
||||
output = captured_output.getvalue()
|
||||
assert package_logger.handlers != [existing_handler]
|
||||
assert len(package_logger.handlers) == 1
|
||||
assert leaked_output.getvalue() == ""
|
||||
assert root_handler in root_logger.handlers
|
||||
assert "exception-secret" not in output
|
||||
assert "private text" not in output
|
||||
assert "[REDACTED EXCEPTION]" in output
|
||||
finally:
|
||||
for handler in package_logger.handlers[:]:
|
||||
package_logger.removeHandler(handler)
|
||||
package_logger.handlers = package_handlers
|
||||
package_logger.setLevel(package_level)
|
||||
package_logger.propagate = package_propagate
|
||||
root_logger.removeHandler(root_handler)
|
||||
|
||||
|
||||
def test_package_import_has_no_runtime_side_effects(tmp_path: Path) -> None:
|
||||
root = tmp_path / "fresh-xdg"
|
||||
source_root = Path(__file__).parents[2] / "src"
|
||||
environment = {
|
||||
"GDC_IMPORT_ROOT": str(root),
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONPATH": str(source_root),
|
||||
}
|
||||
script = """
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["GDC_IMPORT_ROOT"])
|
||||
targets = [root / name for name in ("config", "data", "cache", "runtime")]
|
||||
events = []
|
||||
|
||||
def audit(event, arguments):
|
||||
if event in {"os.mkdir", "os.chmod", "subprocess.Popen", "socket.connect", "sqlite3.connect"}:
|
||||
events.append(event)
|
||||
if event == "open":
|
||||
mode = arguments[1] if len(arguments) > 1 else None
|
||||
flags = arguments[2] if len(arguments) > 2 else 0
|
||||
if isinstance(mode, str) and any(marker in mode for marker in ("w", "a", "x", "+")):
|
||||
events.append("open-write")
|
||||
if isinstance(flags, int) and flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC):
|
||||
events.append("open-write")
|
||||
|
||||
sys.addaudithook(audit)
|
||||
os.environ.update({
|
||||
"HOME": str(root / "home"),
|
||||
"XDG_CONFIG_HOME": str(targets[0]),
|
||||
"XDG_DATA_HOME": str(targets[1]),
|
||||
"XDG_CACHE_HOME": str(targets[2]),
|
||||
"XDG_RUNTIME_DIR": str(targets[3]),
|
||||
})
|
||||
import game_data_capture
|
||||
|
||||
if events or any(path.exists() for path in targets):
|
||||
raise SystemExit(1)
|
||||
"""
|
||||
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-B", "-c", script],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=environment,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from stat import S_IMODE
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.config import ConfigError, ServerConfig, parse_config
|
||||
from game_data_capture.paths import (
|
||||
PathConfigurationError,
|
||||
prepare_private_directories,
|
||||
prepare_private_file,
|
||||
resolve_runtime_paths,
|
||||
)
|
||||
|
||||
|
||||
def test_config_uses_typed_defaults() -> None:
|
||||
config = parse_config({})
|
||||
|
||||
assert config.server == ServerConfig(host="127.0.0.1", port=8765)
|
||||
assert config.lifecycle.instance_name == "shd-capture"
|
||||
assert config.limits.max_queue_items == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize("host", ["0.0.0.0", "192.168.1.2", "localhost", "example.test"])
|
||||
def test_config_rejects_non_loopback_binding(host: str) -> None:
|
||||
with pytest.raises(ConfigError, match="loopback IP literal"):
|
||||
parse_config({"server": {"host": host}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("document", "message"),
|
||||
[
|
||||
({"server": {"port": 80}}, "server.port"),
|
||||
({"limits": {"max_queue_items": 257}}, "limits.max_queue_items"),
|
||||
({"lifecycle": {"instance_name": "SHD Capture"}}, "lifecycle.instance_name"),
|
||||
({"openai": {"api_key_env": "openai-key"}}, "openai.api_key_env"),
|
||||
],
|
||||
)
|
||||
def test_config_rejects_invalid_bounded_values(document: dict[str, object], message: str) -> None:
|
||||
with pytest.raises(ConfigError, match=message):
|
||||
parse_config(document)
|
||||
|
||||
|
||||
def test_xdg_paths_are_resolved_without_writing(tmp_path: Path) -> None:
|
||||
root = tmp_path / "xdg"
|
||||
paths = resolve_runtime_paths(
|
||||
{
|
||||
"HOME": str(tmp_path / "home"),
|
||||
"XDG_CONFIG_HOME": str(root / "config"),
|
||||
"XDG_DATA_HOME": str(root / "data"),
|
||||
"XDG_CACHE_HOME": str(root / "cache"),
|
||||
"XDG_RUNTIME_DIR": str(root / "runtime"),
|
||||
}
|
||||
)
|
||||
|
||||
assert paths.runtime_dir == root / "runtime" / "shd-capture"
|
||||
assert not root.exists()
|
||||
|
||||
|
||||
def test_xdg_paths_reject_relative_runtime_path() -> None:
|
||||
with pytest.raises(PathConfigurationError, match="XDG runtime"):
|
||||
resolve_runtime_paths({"HOME": "/tmp/home", "XDG_RUNTIME_DIR": "relative"})
|
||||
|
||||
|
||||
def test_runtime_directories_and_lock_file_are_owner_only(tmp_path: Path) -> None:
|
||||
paths = resolve_runtime_paths(
|
||||
{
|
||||
"HOME": str(tmp_path),
|
||||
"XDG_CONFIG_HOME": str(tmp_path / "config"),
|
||||
"XDG_DATA_HOME": str(tmp_path / "data"),
|
||||
"XDG_CACHE_HOME": str(tmp_path / "cache"),
|
||||
"XDG_RUNTIME_DIR": str(tmp_path / "runtime"),
|
||||
}
|
||||
)
|
||||
|
||||
prepare_private_directories(paths)
|
||||
prepare_private_file(paths.instance_lock_file)
|
||||
|
||||
assert S_IMODE(paths.runtime_dir.stat().st_mode) == 0o700
|
||||
assert S_IMODE(paths.instance_lock_file.stat().st_mode) == 0o600
|
||||
@@ -0,0 +1,337 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.routing import Mount
|
||||
from starlette.staticfiles import StaticFiles
|
||||
from starlette.types import Scope
|
||||
|
||||
from game_data_capture.config import ServerConfig
|
||||
from game_data_capture.web.app import (
|
||||
CaptureRecord,
|
||||
CaptureSessionStatus,
|
||||
ControlOutcome,
|
||||
MediaDescriptor,
|
||||
MutationOutcome,
|
||||
WebConfig,
|
||||
create_app,
|
||||
)
|
||||
from game_data_capture.web.security import new_bootstrap_token
|
||||
|
||||
_CAPTURE_ID = "capture-00000000000000000000000000000001"
|
||||
_LABEL: dict[str, object] = {
|
||||
"family": "weapon",
|
||||
"view": "detail",
|
||||
"gear_slot": None,
|
||||
"record_types": ["weapon", "weapon_talent"],
|
||||
"confidence": 1.0,
|
||||
"abstention_reason": None,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeControl:
|
||||
armed: bool = False
|
||||
starts: int = 0
|
||||
stops: int = 0
|
||||
|
||||
def status(self) -> CaptureSessionStatus:
|
||||
return CaptureSessionStatus(self.armed, 0, 64)
|
||||
|
||||
def start_session(self) -> ControlOutcome:
|
||||
if self.armed:
|
||||
return "already_armed"
|
||||
self.starts += 1
|
||||
self.armed = True
|
||||
return "started"
|
||||
|
||||
def stop_session(self) -> ControlOutcome:
|
||||
if not self.armed:
|
||||
return "already_stopped"
|
||||
self.stops += 1
|
||||
self.armed = False
|
||||
return "stopped"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeCorpus:
|
||||
records: dict[str, CaptureRecord] = field(
|
||||
default_factory=lambda: {
|
||||
_CAPTURE_ID: CaptureRecord(_CAPTURE_ID, "stored", "2026-07-31T00:00:00Z", None)
|
||||
}
|
||||
)
|
||||
calls: list[str] = field(default_factory=list)
|
||||
media: MediaDescriptor | None = None
|
||||
|
||||
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
|
||||
self.calls.append("list")
|
||||
return tuple(self.records.values())[:limit]
|
||||
|
||||
def get_capture(self, capture_id: str) -> CaptureRecord | None:
|
||||
self.calls.append("get")
|
||||
return self.records.get(capture_id)
|
||||
|
||||
def open_media(self, capture_id: str) -> MediaDescriptor | None:
|
||||
self.calls.append("media")
|
||||
if capture_id not in self.records:
|
||||
return None
|
||||
if self.media is not None:
|
||||
return self.media
|
||||
return MediaDescriptor("image/png", 4, [b"data"])
|
||||
|
||||
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
|
||||
self.calls.append("label")
|
||||
if capture_id not in self.records:
|
||||
return "not_found"
|
||||
old = self.records[capture_id]
|
||||
self.records[capture_id] = CaptureRecord(
|
||||
old.capture_id, old.state, old.received_at, dict(classification)
|
||||
)
|
||||
return "updated"
|
||||
|
||||
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
|
||||
self.calls.append("reject")
|
||||
return "updated" if capture_id in self.records else "not_found"
|
||||
|
||||
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
|
||||
self.calls.append("delete")
|
||||
return "retained" if capture_id in self.records else "not_found"
|
||||
|
||||
|
||||
def _client(control: FakeControl, corpus: FakeCorpus) -> tuple[TestClient, str]:
|
||||
token = new_bootstrap_token()
|
||||
app = create_app(
|
||||
config=WebConfig(ServerConfig()),
|
||||
capture_control=control,
|
||||
corpus=corpus,
|
||||
bootstrap_token=token,
|
||||
)
|
||||
return TestClient(app, base_url="http://127.0.0.1:8765"), token
|
||||
|
||||
|
||||
def _bootstrap(client: TestClient, token: str) -> str:
|
||||
response = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return str(response.json()["csrf_token"])
|
||||
|
||||
|
||||
def _mutation_headers(csrf: str) -> dict[str, str]:
|
||||
return {"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf}
|
||||
|
||||
|
||||
def test_factory_and_authenticated_gets_never_arm_capture() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
|
||||
assert control.starts == 0
|
||||
assert client.get("/health").json() == {"status": "ok"}
|
||||
csrf = _bootstrap(client, token)
|
||||
assert client.get("/api/session").status_code == 200
|
||||
assert client.get("/api/captures").status_code == 200
|
||||
assert client.get(f"/api/captures/{_CAPTURE_ID}").status_code == 200
|
||||
assert control.starts == 0
|
||||
assert csrf
|
||||
|
||||
|
||||
def test_static_contract_is_package_local_and_does_not_accept_a_filesystem_root() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
app = create_app(
|
||||
config=WebConfig(ServerConfig()),
|
||||
capture_control=control,
|
||||
corpus=corpus,
|
||||
bootstrap_token=new_bootstrap_token(),
|
||||
)
|
||||
static_mount = next(
|
||||
route for route in app.routes if isinstance(route, Mount) and route.path == "/static"
|
||||
)
|
||||
|
||||
assert isinstance(static_mount.app, StaticFiles)
|
||||
assert static_mount.app.follow_symlink is False
|
||||
assert static_mount.app.directory is not None
|
||||
assert Path(static_mount.app.directory).name == "static"
|
||||
assert not Path(
|
||||
static_mount.app.get_path(
|
||||
cast(Scope, {"path": "/static/../../etc/passwd", "root_path": ""})
|
||||
)
|
||||
).is_absolute()
|
||||
|
||||
|
||||
def test_bootstrap_cookie_is_host_only_http_only_and_never_echoes_bootstrap_token() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
|
||||
response = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert token not in response.text
|
||||
cookie = response.headers["set-cookie"].lower()
|
||||
assert "httponly" in cookie
|
||||
assert "samesite=strict" in cookie
|
||||
assert "domain=" not in cookie
|
||||
assert token not in cookie
|
||||
assert response.json().keys() == {"csrf_token"}
|
||||
|
||||
|
||||
def test_start_is_explicit_csrf_protected_idempotent_and_rotates_session() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
csrf = _bootstrap(client, token)
|
||||
old_session = str(client.cookies["capture_session"])
|
||||
|
||||
response = client.post(
|
||||
"/api/session/start", headers=_mutation_headers(csrf), json={"confirmation": "START"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["result"] == "started"
|
||||
assert control.starts == 1
|
||||
next_csrf = str(response.json()["csrf_token"])
|
||||
stale = client.post(
|
||||
"/api/session/start", headers=_mutation_headers(csrf), json={"confirmation": "START"}
|
||||
)
|
||||
assert stale.status_code == 403
|
||||
assert control.starts == 1
|
||||
expired_session = client.post(
|
||||
"/api/session/start",
|
||||
headers={**_mutation_headers(next_csrf), "Cookie": f"capture_session={old_session}"},
|
||||
json={"confirmation": "START"},
|
||||
)
|
||||
assert expired_session.status_code == 401
|
||||
assert control.starts == 1
|
||||
duplicate = client.post(
|
||||
"/api/session/start", headers=_mutation_headers(next_csrf), json={"confirmation": "START"}
|
||||
)
|
||||
assert duplicate.status_code == 200
|
||||
assert duplicate.json()["result"] == "already_armed"
|
||||
assert control.starts == 1
|
||||
|
||||
|
||||
def test_label_reject_and_retention_deletion_use_closed_confirmed_contracts() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
csrf = _bootstrap(client, token)
|
||||
|
||||
labeled = client.post(
|
||||
f"/api/captures/{_CAPTURE_ID}/label", headers=_mutation_headers(csrf), json=_LABEL
|
||||
)
|
||||
assert labeled.status_code == 200
|
||||
csrf = str(labeled.json()["csrf_token"])
|
||||
detail = client.get(f"/api/captures/{_CAPTURE_ID}")
|
||||
assert detail.json()["capture"]["classification"] == _LABEL
|
||||
rejected = client.post(
|
||||
f"/api/captures/{_CAPTURE_ID}/reject",
|
||||
headers=_mutation_headers(csrf),
|
||||
json={"confirmation": "REJECT"},
|
||||
)
|
||||
assert rejected.status_code == 200
|
||||
csrf = str(rejected.json()["csrf_token"])
|
||||
denied = client.post(
|
||||
f"/api/captures/{_CAPTURE_ID}/delete",
|
||||
headers=_mutation_headers(csrf),
|
||||
json={"confirmation": "delete"},
|
||||
)
|
||||
assert denied.status_code == 403
|
||||
assert "delete" not in corpus.calls
|
||||
retained = client.post(
|
||||
f"/api/captures/{_CAPTURE_ID}/delete",
|
||||
headers=_mutation_headers(csrf),
|
||||
json={"confirmation": "DELETE"},
|
||||
)
|
||||
assert retained.status_code == 409
|
||||
assert retained.json() == {"error": "retained"}
|
||||
assert corpus.calls.count("label") == 1
|
||||
assert corpus.calls.count("reject") == 1
|
||||
assert corpus.calls.count("delete") == 1
|
||||
|
||||
|
||||
def test_opaque_media_route_has_private_nosniff_response_and_no_path_parameter() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
_bootstrap(client, token)
|
||||
|
||||
response = client.get(f"/api/captures/{_CAPTURE_ID}/media")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content == b"data"
|
||||
assert response.headers["content-type"] == "image/png"
|
||||
assert response.headers["cache-control"] == "private, no-store"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
assert client.get("/api/captures/../../etc/passwd/media").status_code == 404
|
||||
assert corpus.calls.count("media") == 1
|
||||
|
||||
|
||||
class RaisingChunks:
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
raise RuntimeError("raw-secret /private/capture/path")
|
||||
|
||||
|
||||
def test_malformed_media_streams_fail_before_success_headers_or_content() -> None:
|
||||
descriptors = (
|
||||
MediaDescriptor("image/png", 4, [b"abc"]),
|
||||
MediaDescriptor("image/png", 3, [b"abcd"]),
|
||||
MediaDescriptor("image/png", 3, [b""]),
|
||||
MediaDescriptor("image/png", 3, cast(Iterable[bytes], ["bad"])),
|
||||
MediaDescriptor("image/png", 3, RaisingChunks()),
|
||||
)
|
||||
for descriptor in descriptors:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus(media=descriptor)
|
||||
client, token = _client(control, corpus)
|
||||
_bootstrap(client, token)
|
||||
|
||||
response = client.get(f"/api/captures/{_CAPTURE_ID}/media")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {"error": "service_unavailable"}
|
||||
assert "raw-secret" not in response.text
|
||||
assert "/private/capture/path" not in response.text
|
||||
|
||||
|
||||
class FailingCorpus(FakeCorpus):
|
||||
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
|
||||
raise RuntimeError("raw-secret /private/capture/path")
|
||||
|
||||
|
||||
def test_port_exceptions_are_sanitized_without_an_uncaught_client_error() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FailingCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
_bootstrap(client, token)
|
||||
|
||||
response = client.get("/api/captures")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {"error": "service_unavailable"}
|
||||
assert "raw-secret" not in response.text
|
||||
assert "/private/capture/path" not in response.text
|
||||
|
||||
|
||||
def test_taxonomy_payload_rejects_unknown_and_ambiguous_values_before_corpus_mutation() -> None:
|
||||
control = FakeControl()
|
||||
corpus = FakeCorpus()
|
||||
client, token = _client(control, corpus)
|
||||
csrf = _bootstrap(client, token)
|
||||
unsupported = dict(_LABEL)
|
||||
unsupported["view"] = "not-a-view"
|
||||
unsupported["record_types"] = ["gear"]
|
||||
|
||||
response = client.post(
|
||||
f"/api/captures/{_CAPTURE_ID}/label", headers=_mutation_headers(csrf), json=unsupported
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert "label" not in corpus.calls
|
||||
@@ -0,0 +1,321 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from game_data_capture.config import ServerConfig
|
||||
from game_data_capture.web.app import (
|
||||
CaptureCorpusPort,
|
||||
CaptureRecord,
|
||||
CaptureSessionStatus,
|
||||
ControlOutcome,
|
||||
MediaDescriptor,
|
||||
MutationOutcome,
|
||||
WebConfig,
|
||||
create_app,
|
||||
)
|
||||
from game_data_capture.web.security import (
|
||||
LocalAuthority,
|
||||
LocalRequestLimitsMiddleware,
|
||||
RequestLimits,
|
||||
new_bootstrap_token,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Control:
|
||||
start_calls: int = 0
|
||||
|
||||
def status(self) -> CaptureSessionStatus:
|
||||
return CaptureSessionStatus(False, 0, 1)
|
||||
|
||||
def start_session(self) -> ControlOutcome:
|
||||
self.start_calls += 1
|
||||
return "started"
|
||||
|
||||
def stop_session(self) -> ControlOutcome:
|
||||
return "already_stopped"
|
||||
|
||||
|
||||
class Corpus(CaptureCorpusPort):
|
||||
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
|
||||
return ()
|
||||
|
||||
def get_capture(self, capture_id: str) -> CaptureRecord | None:
|
||||
return None
|
||||
|
||||
def open_media(self, capture_id: str) -> MediaDescriptor | None:
|
||||
return None
|
||||
|
||||
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self) -> None:
|
||||
self.value = 0.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.value
|
||||
|
||||
|
||||
def _app(
|
||||
control: Control, *, now: Callable[[], float] | None = None, max_body: int = 16 * 1024
|
||||
) -> tuple[FastAPI, str]:
|
||||
token = new_bootstrap_token()
|
||||
config = WebConfig(ServerConfig(), request_limits=RequestLimits(max_body_bytes=max_body))
|
||||
if now is None:
|
||||
app = create_app(
|
||||
config=config,
|
||||
capture_control=control,
|
||||
corpus=Corpus(),
|
||||
bootstrap_token=token,
|
||||
)
|
||||
else:
|
||||
app = create_app(
|
||||
config=config,
|
||||
capture_control=control,
|
||||
corpus=Corpus(),
|
||||
bootstrap_token=token,
|
||||
now=now,
|
||||
)
|
||||
return app, token
|
||||
|
||||
|
||||
def _bootstrap(client: TestClient, token: str) -> str:
|
||||
result = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
|
||||
)
|
||||
assert result.status_code == 200
|
||||
return str(result.json()["csrf_token"])
|
||||
|
||||
|
||||
def test_host_origin_and_csrf_fail_before_start_side_effect() -> None:
|
||||
control = Control()
|
||||
app, token = _app(control)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
csrf = _bootstrap(client, token)
|
||||
|
||||
bad_host = client.post(
|
||||
"/api/session/start",
|
||||
headers={"Host": "evil.test:8765", "Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf},
|
||||
json={"confirmation": "START"},
|
||||
)
|
||||
missing_origin = client.post(
|
||||
"/api/session/start", headers={"X-CSRF-Token": csrf}, json={"confirmation": "START"}
|
||||
)
|
||||
cross_origin = client.post(
|
||||
"/api/session/start",
|
||||
headers={"Origin": "http://evil.test", "X-CSRF-Token": csrf},
|
||||
json={"confirmation": "START"},
|
||||
)
|
||||
bad_csrf = client.post(
|
||||
"/api/session/start",
|
||||
headers={"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": "x" * 43},
|
||||
json={"confirmation": "START"},
|
||||
)
|
||||
|
||||
assert bad_host.status_code == 400
|
||||
assert missing_origin.status_code == 403
|
||||
assert cross_origin.status_code == 403
|
||||
assert bad_csrf.status_code == 403
|
||||
assert control.start_calls == 0
|
||||
|
||||
|
||||
def test_session_expiry_and_process_restart_invalidate_cookie() -> None:
|
||||
control = Control()
|
||||
clock = Clock()
|
||||
app, token = _app(control, now=clock)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
_bootstrap(client, token)
|
||||
assert client.get("/api/session").status_code == 200
|
||||
clock.value = 901
|
||||
assert client.get("/api/session").status_code == 401
|
||||
|
||||
new_app, _ = _app(control)
|
||||
restarted = TestClient(new_app, base_url="http://127.0.0.1:8765")
|
||||
restarted.cookies.update(client.cookies)
|
||||
assert restarted.get("/api/session").status_code == 401
|
||||
|
||||
|
||||
def test_duplicate_json_and_form_payloads_fail_before_start_side_effect() -> None:
|
||||
control = Control()
|
||||
app, token = _app(control)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
csrf = _bootstrap(client, token)
|
||||
headers = {"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf}
|
||||
|
||||
duplicate_json = client.post(
|
||||
"/api/session/start",
|
||||
headers={**headers, "Content-Type": "application/json"},
|
||||
content=b'{"confirmation":"START","confirmation":"START"}',
|
||||
)
|
||||
form = client.post(
|
||||
"/api/session/start",
|
||||
headers={**headers, "Content-Type": "application/x-www-form-urlencoded"},
|
||||
content=b"confirmation=START&confirmation=START",
|
||||
)
|
||||
|
||||
assert duplicate_json.status_code == 403
|
||||
assert form.status_code == 403
|
||||
assert control.start_calls == 0
|
||||
|
||||
|
||||
def test_ipv6_host_requires_bracketed_exact_authority() -> None:
|
||||
control = Control()
|
||||
token = new_bootstrap_token()
|
||||
app = create_app(
|
||||
config=WebConfig(ServerConfig(host="::1", port=8765)),
|
||||
capture_control=control,
|
||||
corpus=Corpus(),
|
||||
bootstrap_token=token,
|
||||
)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
|
||||
accepted = client.get("/health", headers={"Host": "[::1]:8765"})
|
||||
rejected = client.get("/health", headers={"Host": "::1:8765"})
|
||||
|
||||
assert accepted.status_code == 200
|
||||
assert rejected.status_code == 400
|
||||
|
||||
|
||||
def test_all_success_and_error_responses_have_local_only_security_headers() -> None:
|
||||
control = Control()
|
||||
app, _ = _app(control)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
responses = (
|
||||
client.get("/health"),
|
||||
client.get("/health", headers={"Host": "evil.test:8765"}),
|
||||
client.get("/api/session"),
|
||||
)
|
||||
for response in responses:
|
||||
csp = response.headers["content-security-policy"]
|
||||
assert "default-src 'self'" in csp
|
||||
assert "script-src 'self'" in csp
|
||||
assert "style-src 'self'" in csp
|
||||
assert "font-src 'self'" in csp
|
||||
assert "img-src 'self'" in csp
|
||||
assert "media-src 'self'" in csp
|
||||
assert "connect-src 'self'" in csp
|
||||
assert "object-src 'none'" in csp
|
||||
assert "base-uri 'none'" in csp
|
||||
assert "frame-ancestors 'none'" in csp
|
||||
assert response.headers["referrer-policy"] == "no-referrer"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
assert response.headers["x-frame-options"] == "DENY"
|
||||
|
||||
|
||||
async def _call_raw(app: ASGIApp, scope: Scope, messages: list[Message]) -> list[Message]:
|
||||
sent: list[Message] = []
|
||||
|
||||
async def receive() -> Message:
|
||||
return messages.pop(0)
|
||||
|
||||
async def send(message: Message) -> None:
|
||||
sent.append(message)
|
||||
|
||||
await app(scope, receive, send)
|
||||
return sent
|
||||
|
||||
|
||||
def _scope(path: str, headers: list[tuple[bytes, bytes]], *, method: str = "POST") -> Scope:
|
||||
return {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": method,
|
||||
"scheme": "http",
|
||||
"path": path,
|
||||
"raw_path": path.encode(),
|
||||
"query_string": b"",
|
||||
"headers": headers,
|
||||
"client": ("127.0.0.1", 50000),
|
||||
"server": ("127.0.0.1", 8765),
|
||||
}
|
||||
|
||||
|
||||
def test_declared_streamed_duplicate_and_disconnect_requests_fail_before_start() -> None:
|
||||
control = Control()
|
||||
app, _ = _app(control, max_body=32)
|
||||
base_headers = [(b"host", b"127.0.0.1:8765"), (b"content-type", b"application/json")]
|
||||
|
||||
declared = asyncio.run(
|
||||
_call_raw(
|
||||
app,
|
||||
_scope("/api/session/start", [*base_headers, (b"content-length", b"33")]),
|
||||
[{"type": "http.request", "body": b"", "more_body": False}],
|
||||
)
|
||||
)
|
||||
duplicated = asyncio.run(
|
||||
_call_raw(
|
||||
app,
|
||||
_scope("/api/session/start", [*base_headers, (b"host", b"127.0.0.1:8765")]),
|
||||
[{"type": "http.request", "body": b"{}", "more_body": False}],
|
||||
)
|
||||
)
|
||||
streamed = asyncio.run(
|
||||
_call_raw(
|
||||
app,
|
||||
_scope(
|
||||
"/api/bootstrap",
|
||||
[*base_headers, (b"origin", b"http://127.0.0.1:8765")],
|
||||
),
|
||||
[
|
||||
{"type": "http.request", "body": b'{"token":"', "more_body": True},
|
||||
{"type": "http.request", "body": b"x" * 24, "more_body": False},
|
||||
],
|
||||
)
|
||||
)
|
||||
disconnected = asyncio.run(
|
||||
_call_raw(
|
||||
app,
|
||||
_scope(
|
||||
"/api/bootstrap",
|
||||
[*base_headers, (b"origin", b"http://127.0.0.1:8765")],
|
||||
),
|
||||
[{"type": "http.disconnect"}],
|
||||
)
|
||||
)
|
||||
|
||||
assert declared[0]["status"] == 413
|
||||
assert duplicated[0]["status"] == 400
|
||||
assert streamed[0]["status"] == 413
|
||||
assert disconnected[0]["status"] == 400
|
||||
assert control.start_calls == 0
|
||||
|
||||
|
||||
def test_post_completion_disconnect_is_normal_for_streaming_response_lifecycle() -> None:
|
||||
async def streaming_lifecycle(scope: Scope, receive: Receive, send: Send) -> None:
|
||||
await receive()
|
||||
assert (await receive())["type"] == "http.disconnect"
|
||||
await send({"type": "http.response.start", "status": 200, "headers": []})
|
||||
await send({"type": "http.response.body", "body": b"ok"})
|
||||
|
||||
guarded = LocalRequestLimitsMiddleware(
|
||||
streaming_lifecycle,
|
||||
authority=LocalAuthority("127.0.0.1", 8765),
|
||||
limits=RequestLimits(),
|
||||
)
|
||||
sent = asyncio.run(
|
||||
_call_raw(
|
||||
guarded,
|
||||
_scope("/stream", [(b"host", b"127.0.0.1:8765")], method="GET"),
|
||||
[
|
||||
{"type": "http.request", "body": b"", "more_body": False},
|
||||
{"type": "http.disconnect"},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
assert sent[0]["status"] == 200
|
||||
Reference in New Issue
Block a user