Add game data capture vertical slice

This commit is contained in:
jb
2026-07-31 14:34:24 -04:00
parent 5859f84034
commit fee4dc3a78
75 changed files with 12654 additions and 5 deletions
@@ -0,0 +1,458 @@
from __future__ import annotations
import sqlite3
import threading
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
from game_data_capture.storage import CaptureCorpus, CaptureRepository, ContentAddressedBlobStore
from game_data_capture.storage.blobs import BlobCorruptionError, BlobLockError
from game_data_capture.storage.database import StorageDatabase
from game_data_capture.storage.repository import StorageError
def _label(*, view: str = "detail", records: list[str] | None = None) -> dict[str, object]:
return {
"family": "weapon",
"view": view,
"gear_slot": None,
"record_types": ["weapon"] if records is None else records,
"confidence": 1.0,
"abstention_reason": None,
}
@pytest.fixture
def corpus(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository, CaptureCorpus]:
root = (tmp_path / "utility").resolve()
database = StorageDatabase(
root,
(Path(__file__).parents[3] / "migrations").resolve(),
)
database.migrate()
blobs = ContentAddressedBlobStore(root)
repository = CaptureRepository(database, blobs)
return database, repository, CaptureCorpus(database, blobs, max_media_bytes=1024)
def _receive(
repository: CaptureRepository, capture_id: str, token: str, payload: bytes = b"png"
) -> None:
repository.receive(capture_id, token, [payload], media_type="image/png", max_bytes=1024)
def _assert_media_unavailable(port: CaptureCorpus, capture_id: str) -> None:
media = port.open_media(capture_id)
assert media is not None
with pytest.raises(BlobCorruptionError):
next(iter(media.chunks))
def _commit_deletion_intent(database: StorageDatabase, capture_id: str) -> None:
with database.connect() as connection:
row = connection.execute(
"SELECT c.blob_sha256, c.state, c.received_at, c.receipt_token, c.failure_reason, "
"c.revision, b.byte_size FROM captures AS c JOIN blobs AS b "
"ON b.sha256 = c.blob_sha256 WHERE c.capture_id = ?",
(capture_id,),
).fetchone()
assert row is not None
connection.execute("BEGIN IMMEDIATE")
connection.execute(
"INSERT INTO capture_deletion_intents(blob_sha256, capture_id, byte_size, "
"capture_state, received_at, receipt_token, failure_reason, revision, created_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
row["blob_sha256"],
capture_id,
row["byte_size"],
row["state"],
row["received_at"],
row["receipt_token"],
row["failure_reason"],
row["revision"],
"time",
),
)
connection.execute("DELETE FROM captures WHERE capture_id = ?", (capture_id,))
connection.execute("COMMIT")
def test_safe_bounded_listing_detail_and_verified_media(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
_, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
_receive(repository, "capture-0002", "receipt-0002")
records = port.list_captures(limit=1)
assert len(records) == 1 and records[0].capture_id == "capture-0002"
assert port.get_capture("../capture-0001") is None
assert port.open_media("../capture-0001") is None
media = port.open_media("capture-0001")
assert media is not None
assert media.media_type == "image/png" and media.byte_size == 3
chunks = media.chunks
assert next(iter(chunks)) == b"png"
close = getattr(chunks, "close", None)
assert callable(close)
close()
for invalid in (0, 257, True):
with pytest.raises(ValueError):
port.list_captures(limit=invalid)
def test_labels_preserve_prior_values_across_restart_and_reject_is_idempotent(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
first = _label()
second = _label(view="blueprint", records=["weapon_talent"])
with pytest.raises(ValueError):
port.label("capture-0001", {"family": "weapon"})
assert port.label("capture-0001", first) == "updated"
assert port.label("capture-0001", first) == "updated"
assert port.label("capture-0001", second) == "updated"
restarted = CaptureCorpus(
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
)
record = restarted.get_capture("capture-0001")
assert record is not None and record.classification == second
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
assert restarted.label("capture-0001", first) == "conflict"
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM capture_rejections").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
def test_concurrent_label_writes_serialize_without_losing_prior_value(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
labels = (_label(), _label(view="blueprint", records=["weapon_talent"]))
with ThreadPoolExecutor(max_workers=2) as executor:
outcomes = list(executor.map(lambda value: port.label("capture-0001", value), labels))
assert outcomes == ["updated", "updated"]
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
def test_canonical_equivalent_labels_do_not_append_history(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
first = _label(records=["weapon", "weapon_talent"])
equivalent = _label(records=["weapon_talent", "weapon"])
equivalent["confidence"] = 1
assert port.label("capture-0001", first) == "updated"
assert port.label("capture-0001", equivalent) == "updated"
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
def test_rejected_or_evidence_bearing_captures_are_retained(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
_, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
assert port.reject_preserving_evidence("capture-0001") == "updated"
assert port.delete_with_retention("capture-0001") == "retained"
_receive(repository, "capture-0002", "receipt-0002")
repository.append_ocr_run("run-0002", "capture-0002", "ocr", "1", {})
assert port.delete_with_retention("capture-0002") == "retained"
assert port.delete_with_retention("bad") == "not_found"
def test_label_and_job_evidence_retain_capture(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
assert port.label("capture-0001", _label()) == "updated"
assert port.delete_with_retention("capture-0001") == "retained"
_receive(repository, "capture-0002", "receipt-0002")
with database.connect() as connection:
connection.execute(
"INSERT INTO jobs(job_id, capture_id, kind, state, max_attempts) "
"VALUES (?, ?, ?, ?, ?)",
("job-0002", "capture-0002", "ocr", "queued", 1),
)
assert port.delete_with_retention("capture-0002") == "retained"
def test_receive_translates_put_lock_failure_without_capture(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
monkeypatch: pytest.MonkeyPatch,
) -> None:
database, repository, _ = corpus
def locked(_: object, *, max_bytes: int) -> tuple[str, int]:
raise BlobLockError("locked")
monkeypatch.setattr(repository.blobs, "put", locked)
with pytest.raises(StorageError, match="unavailable"):
_receive(repository, "capture-0001", "receipt-0001")
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
@pytest.mark.parametrize(
("state", "failure_reason"),
(("failed", None), ("cancelled", None), ("stored", "capture failure"), ("unknown", None)),
)
def test_non_disposable_capture_rows_are_retained(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
state: str,
failure_reason: str | None,
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
with database.connect() as connection:
connection.execute(
"UPDATE captures SET state = ?, failure_reason = ? WHERE capture_id = ?",
(state, failure_reason, "capture-0001"),
)
assert port.delete_with_retention("capture-0001") == "retained"
def test_capture_lineage_and_failure_evidence_are_retained(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
repository.append_lineage("lineage-0001", "capture-0001", "parent-0001", "derived")
assert port.delete_with_retention("capture-0001") == "retained"
_receive(repository, "capture-0002", "receipt-0002")
repository.record_failure("failure-0002", "capture", "capture-0002", "fault", "bounded")
assert port.delete_with_retention("capture-0002") == "retained"
_receive(repository, "capture-0003", "receipt-0003")
repository.append_lineage("lineage-0003", "child-0003", "capture-0003", "derived")
assert port.delete_with_retention("capture-0003") == "retained"
def test_shared_blob_deletion_keeps_remaining_capture_media(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
_, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001", b"shared")
_receive(repository, "capture-0002", "receipt-0002", b"shared")
assert port.delete_with_retention("capture-0001") == "updated"
assert port.get_capture("capture-0001") is None
media = port.open_media("capture-0002")
assert media is not None and b"".join(media.chunks) == b"shared"
with repository.database.connect() as connection:
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
def test_unshared_deletion_removes_media_metadata_and_intent(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
assert row is not None
path = repository.blobs._path(str(row["blob_sha256"]))
assert port.delete_with_retention("capture-0001") == "updated"
assert not path.exists()
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
)
def test_committed_intent_before_unlink_blocks_replay_then_finalizes(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
assert row is not None
path = repository.blobs._path(str(row["blob_sha256"]))
_commit_deletion_intent(database, "capture-0001")
assert path.exists()
with pytest.raises(StorageError, match="deletion"):
_receive(repository, "capture-0002", "receipt-0001")
restarted = CaptureCorpus(
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
)
assert restarted.reconcile_deletions() == 1
assert not path.exists()
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
)
def test_interleaved_reinstall_waits_for_deletion_finalization(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
monkeypatch: pytest.MonkeyPatch,
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001", b"shared")
unlinked = threading.Event()
release = threading.Event()
original = repository.blobs.delete_verified
outcomes: list[object] = []
def pause(digest: str, *, expected_size: int) -> bool:
result = original(digest, expected_size=expected_size)
unlinked.set()
assert release.wait(0.5)
return result
monkeypatch.setattr(repository.blobs, "delete_verified", pause)
deletion = threading.Thread(
target=lambda: outcomes.append(port.delete_with_retention("capture-0001"))
)
deletion.start()
assert unlinked.wait(0.5)
receipt = threading.Thread(
target=lambda: outcomes.append(
repository.receive(
"capture-0002", "receipt-0002", [b"shared"], media_type="image/png", max_bytes=1024
)
)
)
receipt.start()
release.set()
deletion.join(1)
receipt.join(1)
assert not deletion.is_alive() and not receipt.is_alive()
assert set(outcomes) == {"updated", "capture-0002"}
with database.connect() as connection:
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
)
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
media = port.open_media("capture-0002")
assert media is not None and b"".join(media.chunks) == b"shared"
def test_unlink_then_sqlite_finalization_failure_recovers(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
row = database.connect().execute("SELECT sha256, byte_size FROM blobs").fetchone()
assert row is not None
path = repository.blobs._path(str(row["sha256"]))
_commit_deletion_intent(database, "capture-0001")
repository.blobs.delete_verified(str(row["sha256"]), expected_size=int(row["byte_size"]))
assert not path.exists()
with database.connect() as connection:
connection.execute(
"CREATE TRIGGER fail_finalization BEFORE DELETE ON capture_deletion_intents "
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
)
with pytest.raises(sqlite3.IntegrityError, match="fault"):
port.reconcile_deletions()
assert port.get_capture("capture-0001") is None
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 1
)
connection.execute("DROP TRIGGER fail_finalization")
restarted = CaptureCorpus(
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
)
assert restarted.reconcile_deletions() == 1
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
)
assert not path.exists()
def test_corrupt_missing_and_symlink_media_fail_closed(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], tmp_path: Path
) -> None:
_, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
with repository.database.connect() as connection:
row = connection.execute("SELECT blob_sha256 FROM captures").fetchone()
assert row is not None
path = repository.blobs._path(str(row["blob_sha256"]))
path.write_bytes(b"x" * 2048)
_assert_media_unavailable(port, "capture-0001")
path.unlink()
_assert_media_unavailable(port, "capture-0001")
path.symlink_to(tmp_path / "outside")
_assert_media_unavailable(port, "capture-0001")
def test_label_transaction_rolls_back_on_evidence_failure(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
assert port.label("capture-0001", _label()) == "updated"
with database.connect() as connection:
connection.execute(
"CREATE TRIGGER fail_label_evidence BEFORE INSERT ON capture_label_evidence "
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
)
with pytest.raises(sqlite3.IntegrityError, match="fault"):
port.label(
"capture-0001",
_label(view="blueprint", records=["weapon_talent"]),
)
assert port.get_capture("capture-0001") is not None
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
stored = connection.execute("SELECT classification_json FROM capture_labels").fetchone()
assert stored is not None and '"view":"detail"' in stored["classification_json"]
def test_deletion_intent_recovers_after_unlink_failure(
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
monkeypatch: pytest.MonkeyPatch,
) -> None:
database, repository, port = corpus
_receive(repository, "capture-0001", "receipt-0001")
with database.connect() as connection:
original = tuple(
connection.execute(
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
"failure_reason, revision FROM captures"
).fetchone()
)
def fail_unlink(_: str, *, expected_size: int) -> bool:
raise BlobCorruptionError("injected fault")
monkeypatch.setattr(repository.blobs, "delete_verified", fail_unlink)
assert port.delete_with_retention("capture-0001") == "retained"
with database.connect() as connection:
assert (
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
)
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
assert (
tuple(
connection.execute(
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
"failure_reason, revision FROM captures"
).fetchone()
)
== original
)
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
media = port.open_media("capture-0001")
assert media is not None and b"".join(media.chunks) == b"png"
@@ -0,0 +1,147 @@
from __future__ import annotations
import shutil
import sqlite3
from pathlib import Path
import pytest
from game_data_capture.storage.database import MigrationError, StorageDatabase
def _migrations(tmp_path: Path) -> Path:
source = Path(__file__).parents[3] / "migrations" / "001_initial.sql"
destination = tmp_path / "migrations"
destination.mkdir()
shutil.copy(source, destination / source.name)
return destination
def test_connection_pragmas_and_private_modes(tmp_path: Path) -> None:
root = (tmp_path / "utility").resolve()
database = StorageDatabase(root, _migrations(tmp_path).resolve())
database.migrate()
with database.connect() as connection:
connection.execute("BEGIN IMMEDIATE")
connection.execute("CREATE TABLE IF NOT EXISTS live_wal(value INTEGER)")
assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1
assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000
assert connection.execute("PRAGMA synchronous").fetchone()[0] == 2
for path in (
database.path.with_name("capture.sqlite3-wal"),
database.path.with_name("capture.sqlite3-shm"),
):
if path.exists():
assert path.stat().st_mode & 0o077 == 0
connection.execute("COMMIT")
assert root.stat().st_mode & 0o777 == 0o700
assert database.path.stat().st_mode & 0o777 == 0o600
def test_database_symlink_is_rejected(tmp_path: Path) -> None:
root = (tmp_path / "utility").resolve()
root.mkdir()
outside = tmp_path / "outside"
outside.write_text("x")
(root / "capture.sqlite3").symlink_to(outside)
with pytest.raises(MigrationError):
StorageDatabase(root, _migrations(tmp_path).resolve()).connect()
def test_migration_history_and_failure_rollback(tmp_path: Path) -> None:
root = (tmp_path / "utility").resolve()
migrations = _migrations(tmp_path).resolve()
database = StorageDatabase(root, migrations)
database.migrate()
first = migrations / "001_initial.sql"
first.write_text(first.read_text() + "\n-- changed\n")
with pytest.raises(MigrationError):
database.migrate()
first.write_text((Path(__file__).parents[3] / "migrations" / "001_initial.sql").read_text())
bad = migrations / "002_bad.sql"
bad.write_text("CREATE TABLE partial_test(value INTEGER); BAD SQL;")
with pytest.raises(sqlite3.OperationalError):
database.migrate()
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 1
assert (
connection.execute(
"SELECT name FROM sqlite_master WHERE name='partial_test'"
).fetchone()
is None
)
bad.write_text("CREATE TABLE partial_test(value INTEGER);")
database.migrate()
database.migrate()
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
def test_migration_removal_rename_and_duplicate_rejected(tmp_path: Path) -> None:
root = (tmp_path / "utility").resolve()
migrations = _migrations(tmp_path).resolve()
database = StorageDatabase(root, migrations)
database.migrate()
path = migrations / "001_initial.sql"
path.rename(migrations / "001_renamed.sql")
with pytest.raises(MigrationError):
database.migrate()
(migrations / "001_renamed.sql").rename(path)
path.unlink()
with pytest.raises(MigrationError):
database.migrate()
shutil.copy(Path(__file__).parents[3] / "migrations" / "001_initial.sql", path)
shutil.copy(path, migrations / "001_duplicate.sql")
with pytest.raises(MigrationError):
database.migrate()
def test_corpus_migration_preserves_populated_initial_database(tmp_path: Path) -> None:
root = (tmp_path / "utility").resolve()
migrations = _migrations(tmp_path).resolve()
database = StorageDatabase(root, migrations)
database.migrate()
with database.connect() as connection:
connection.execute(
"INSERT INTO blobs(sha256, byte_size, media_type, created_at) VALUES (?, 1, ?, ?)",
("0" * 64, "image/png", "time"),
)
connection.execute(
"INSERT INTO captures(capture_id, blob_sha256, state, received_at, receipt_token) "
"VALUES (?, ?, ?, ?, ?)",
("capture-0001", "0" * 64, "stored", "time", "receipt-0001"),
)
source = Path(__file__).parents[3] / "migrations" / "002_capture_corpus.sql"
shutil.copy(source, migrations / source.name)
database.migrate()
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 0
assert tuple(
connection.execute(
"SELECT sha256, byte_size, media_type, created_at FROM blobs"
).fetchone()
) == ("0" * 64, 1, "image/png", "time")
assert tuple(
connection.execute(
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
"failure_reason, revision FROM captures"
).fetchone()
) == ("capture-0001", "0" * 64, "stored", "time", "receipt-0001", None, 0)
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
migration = connection.execute(
"SELECT filename, checksum FROM schema_migrations WHERE version = 2"
).fetchone()
assert migration is not None
assert migration["filename"] == "002_capture_corpus.sql"
assert isinstance(migration["checksum"], str) and len(migration["checksum"]) == 64
names = {row["name"] for row in connection.execute("SELECT name FROM sqlite_master")}
assert {
"captures_blob_sha256",
"jobs_capture_id",
"ocr_runs_capture_id",
"runtime_failures_capture_subject",
"lineage_child_id",
"lineage_parent_id",
}.issubset(names)
@@ -0,0 +1,117 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
from game_data_capture.pipeline.jobs import JobLeases
from game_data_capture.storage.database import StorageDatabase
from game_data_capture.storage.repository import StorageError
@pytest.fixture
def database(tmp_path: Path) -> StorageDatabase:
database = StorageDatabase(
(tmp_path / "utility").resolve(), (Path(__file__).parents[3] / "migrations").resolve()
)
database.migrate()
return database
def _expire(database: StorageDatabase, job_id: str) -> None:
with database.connect() as connection:
connection.execute(
"UPDATE jobs SET lease_expires_at = ? WHERE job_id = ?",
((datetime.now(UTC) - timedelta(seconds=1)).isoformat(), job_id),
)
def test_job_validation_and_exclusive_ownership(database: StorageDatabase) -> None:
jobs = JobLeases(database)
for job_id, kind, attempts in (
("!", "ocr", 1),
("job", "bad", 1),
("job", "ocr", True),
("job", "ocr", 0),
):
with pytest.raises(ValueError):
jobs.create(job_id, kind, max_attempts=attempts)
jobs.create("job-1", "ocr")
for duration in (0, 301, True):
with pytest.raises(ValueError):
jobs.acquire("owner-1", duration)
with pytest.raises(ValueError):
jobs.acquire("!", 30)
assert jobs.acquire("owner-1", 30) == "job-1"
assert jobs.acquire("owner-2", 30) is None
with pytest.raises(StorageError):
jobs.renew("job-1", "owner-2", 30)
with pytest.raises(StorageError):
jobs.finish("job-1", "owner-2", "succeeded")
for duration in (0, 301, True):
with pytest.raises(ValueError):
jobs.renew("job-1", "owner-1", duration)
def test_expiry_recovery_exhaustion_and_terminal_jobs(database: StorageDatabase) -> None:
jobs = JobLeases(database)
jobs.create("retry", "ocr", max_attempts=2)
assert jobs.acquire("owner-1", 30) == "retry"
_expire(database, "retry")
with pytest.raises(StorageError):
jobs.renew("retry", "owner-1", 30)
with pytest.raises(StorageError):
jobs.finish("retry", "owner-1", "succeeded")
assert jobs.recover() == 1
with database.connect() as connection:
assert tuple(
connection.execute(
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id='retry'"
).fetchone()
) == ("queued", None, None)
assert JobLeases(database).acquire("owner-2", 30) == "retry"
with database.connect() as connection:
assert tuple(
connection.execute(
"SELECT attempt, state, lease_owner FROM jobs WHERE job_id='retry'"
).fetchone()
) == (2, "running", "owner-2")
jobs.create("exhaust", "ocr", max_attempts=1)
assert jobs.acquire("owner-3", 30) == "exhaust"
_expire(database, "exhaust")
assert jobs.recover() == 1
with database.connect() as connection:
assert tuple(
connection.execute(
"SELECT state, attempt, lease_owner, lease_expires_at "
"FROM jobs WHERE job_id='exhaust'"
).fetchone()
) == ("failed", 1, None, None)
assert jobs.acquire("owner-4", 30) is None
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
jobs.create(job_id, "ocr")
assert jobs.acquire("owner-5", 30) == job_id
jobs.finish(job_id, "owner-5", target)
with database.connect() as connection:
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
assert tuple(
connection.execute(
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id = ?",
(job_id,),
).fetchone()
) == (target, None, None)
def test_retry_wait_finish_rejected_without_changing_lease(database: StorageDatabase) -> None:
jobs = JobLeases(database)
jobs.create("job-1", "ocr")
jobs.acquire("owner-1", 30)
with pytest.raises(ValueError):
jobs.finish("job-1", "owner-1", "retry_wait")
with database.connect() as connection:
assert tuple(
connection.execute(
"SELECT state, lease_owner FROM jobs WHERE job_id='job-1'"
).fetchone()
) == ("running", "owner-1")
@@ -0,0 +1,233 @@
from __future__ import annotations
import hashlib
import json
import sqlite3
from collections.abc import Iterator
from pathlib import Path
import pytest
from game_data_capture.pipeline.jobs import JobLeases
from game_data_capture.storage.blobs import ContentAddressedBlobStore
from game_data_capture.storage.database import StorageDatabase
from game_data_capture.storage.repository import CaptureRepository, StorageError
@pytest.fixture
def storage(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository]:
root = (tmp_path / "utility").resolve()
migrations = Path(__file__).parents[3] / "migrations"
database = StorageDatabase(root, migrations.resolve())
database.migrate()
return database, CaptureRepository(database, ContentAddressedBlobStore(root))
def test_identical_occurrences_are_deduplicated_and_receipts_survive_restart(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
assert (
repository.receive(
"capture-1", "receipt-1", [b"same"], media_type="image/png", max_bytes=100
)
== "capture-1"
)
assert (
repository.receive(
"capture-2", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
)
== "capture-2"
)
# A simulated restart uses fresh repository objects and a repeated intake receipt.
restarted = CaptureRepository(database, ContentAddressedBlobStore(database.root))
assert (
restarted.receive(
"capture-other", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
)
== "capture-2"
)
assert restarted.report()["blobs"] == 1
assert restarted.report()["captures"] == 2
def test_capture_never_references_missing_blob_and_cas_rejects_stale(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"complete"], media_type="image/png", max_bytes=100
)
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
assert row is not None and repository.blobs.verify(str(row["blob_sha256"])) == len(b"complete")
assert repository.transition("capture-1", "queued", 0) == 1
with pytest.raises(StorageError):
repository.transition("capture-1", "failed", 0)
def test_reprocessing_is_append_only_and_leases_enforce_owner(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
)
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {"raw": "one"})
repository.append_ocr_run("run-2", "capture-1", "ocr", "2", {"raw": "two"})
assert database.connect().execute("SELECT count(*) FROM ocr_runs").fetchone()[0] == 2
jobs = JobLeases(database)
jobs.create("job-1", "ocr", "capture-1")
assert jobs.acquire("worker-a", 30) == "job-1"
with pytest.raises(StorageError):
jobs.finish("job-1", "worker-b", "succeeded")
jobs.finish("job-1", "worker-a", "succeeded")
def test_replayed_receipt_does_not_consume_payload(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
)
consumed = [False]
class MustNotConsume:
def __iter__(self) -> Iterator[bytes]:
consumed[0] = True
raise AssertionError("receipt replay consumed payload")
assert (
repository.receive(
"capture-2", "receipt-1", MustNotConsume(), media_type="image/png", max_bytes=100
)
== "capture-1"
)
assert not consumed[0]
def test_every_provenance_kind_is_insert_only(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
)
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
repository.append_region("region-1", "run-1", {})
repository.append_classification("class-1", "run-1", {})
repository.append_candidate("candidate-1", "run-1", {})
repository.append_routing("route-1", "candidate-1", {})
repository.append_review("review-1", "candidate-1", {})
repository.append_lineage("lineage-1", "child-1", "parent-1", "derived")
repository.append_export("export-1", "ready", {})
with pytest.raises(sqlite3.IntegrityError):
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
payloads = {
"ocr_runs": "payload_json",
"regions": "region_json",
"classifications": "classification_json",
"extraction_candidates": "candidate_json",
"routing_decisions": "decision_json",
"review_events": "event_json",
"exports": "manifest_json",
}
with database.connect() as connection:
for table in (
"ocr_runs",
"regions",
"classifications",
"extraction_candidates",
"routing_decisions",
"review_events",
"lineage",
"exports",
):
assert connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0] == 1
for table, column in payloads.items():
json.loads(connection.execute(f"SELECT {column} FROM {table}").fetchone()[0])
assert (
connection.execute("SELECT count(*) FROM ocr_runs WHERE run_id='run-1'").fetchone()[0]
== 1
)
def test_failure_sanitization_and_provenance_bounds(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
)
with pytest.raises(ValueError):
repository.append_ocr_run("run", "capture-1", "", "1", {})
with pytest.raises(ValueError):
repository.append_ocr_run("run", "capture-1", "x" * 101, "1", {})
with pytest.raises(ValueError):
repository.append_lineage("line", "child", "parent", "")
with pytest.raises(ValueError):
repository.append_export("export", "", {})
with pytest.raises(ValueError):
repository.append_candidate("candidate", "run", "x" * 1_000_001)
long_token = "x" * 600
short_message = (
"sk-secret123 Authorization: Bearer auth-secret token=token-secret secret=hidden "
"password=pw-secret api_key=key-secret /home/agent/private.png\n"
)
repository.record_failure("failure-1", "capture", "capture-1", "failure-code", short_message)
repository.record_failure("failure-2", "capture", "capture-1", "failure-code", long_token)
with database.connect() as connection:
rows = list(
connection.execute("SELECT code, message FROM runtime_failures ORDER BY failure_id")
)
assert len(rows) == 2
for row in rows:
assert len(row["message"]) <= 512 and len(row["code"]) <= 64
assert "\n" not in row["message"]
for value in (
"sk-secret123",
"auth-secret",
"token-secret",
"hidden",
"pw-secret",
"key-secret",
"/home/agent/private.png",
"\n",
):
assert value not in rows[0]["message"]
assert long_token not in rows[1]["message"]
with pytest.raises(ValueError):
repository.record_failure("failure-2", "BAD", "capture-1", "code", "message")
def test_complete_blob_after_capture_insert_failure_is_purgeable_orphan(
storage: tuple[StorageDatabase, CaptureRepository],
) -> None:
database, repository = storage
repository.receive(
"capture-1", "receipt-1", [b"payload-a"], media_type="image/png", max_bytes=100
)
payload_b = b"payload-b"
digest_b = hashlib.sha256(payload_b).hexdigest()
with pytest.raises(sqlite3.IntegrityError):
repository.receive(
"capture-1", "receipt-2", [payload_b], media_type="image/png", max_bytes=100
)
assert repository.blobs.verify(digest_b) == len(payload_b)
with database.connect() as connection:
assert (
connection.execute(
"SELECT count(*) FROM blobs WHERE sha256 = ?", (digest_b,)
).fetchone()[0]
== 0
)
assert (
connection.execute(
"SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest_b,)
).fetchone()[0]
== 0
)
assert repository.blobs._path(digest_b).relative_to(
repository.blobs.root
) in repository.blobs.purge_dry_run(set())
@@ -0,0 +1,195 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from fastapi.testclient import TestClient
from game_data_capture.config import ServerConfig
from game_data_capture.web.app import (
CaptureRecord,
CaptureSessionStatus,
ControlOutcome,
MediaDescriptor,
MutationOutcome,
WebConfig,
create_app,
)
from game_data_capture.web.security import new_bootstrap_token
_ID = "capture-00000000000000000000000000000002"
@dataclass
class CaptureControl:
armed: bool = False
def status(self) -> CaptureSessionStatus:
return CaptureSessionStatus(self.armed, 1, 64)
def start_session(self) -> ControlOutcome:
self.armed = True
return "started"
def stop_session(self) -> ControlOutcome:
self.armed = False
return "stopped"
class Corpus:
def __init__(self) -> None:
self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", None)
self.rejected = False
self.deleted = False
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
return (self.record,)
def get_capture(self, capture_id: str) -> CaptureRecord | None:
return self.record if capture_id == _ID else None
def open_media(self, capture_id: str) -> MediaDescriptor | None:
return MediaDescriptor("image/png", 3, [b"png"]) if capture_id == _ID else None
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
if capture_id != _ID:
return "not_found"
self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", dict(classification))
return "updated"
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
if capture_id != _ID:
return "not_found"
self.rejected = True
return "updated"
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
if capture_id != _ID:
return "not_found"
self.deleted = True
return "updated"
def test_private_capture_labeling_flow_preserves_rejection_before_retention_delete() -> None:
token = new_bootstrap_token()
control = CaptureControl()
corpus = Corpus()
client = TestClient(
create_app(
config=WebConfig(ServerConfig()),
capture_control=control,
corpus=corpus,
bootstrap_token=token,
),
base_url="http://127.0.0.1:8765",
)
origin = "http://127.0.0.1:8765"
csrf = str(
client.post("/api/bootstrap", headers={"Origin": origin}, json={"token": token}).json()[
"csrf_token"
]
)
label = {
"family": "gear",
"view": "detail",
"gear_slot": "Mask",
"record_types": ["gear", "brand_set"],
"confidence": 1.0,
"abstention_reason": None,
}
response = client.post(
f"/api/captures/{_ID}/label", headers={"Origin": origin, "X-CSRF-Token": csrf}, json=label
)
assert response.status_code == 200
csrf = str(response.json()["csrf_token"])
assert client.get(f"/api/captures/{_ID}").json()["capture"]["classification"] == label
response = client.post(
f"/api/captures/{_ID}/reject",
headers={"Origin": origin, "X-CSRF-Token": csrf},
json={"confirmation": "REJECT"},
)
assert response.status_code == 200
csrf = str(response.json()["csrf_token"])
assert corpus.rejected is True
response = client.post(
f"/api/captures/{_ID}/delete",
headers={"Origin": origin, "X-CSRF-Token": csrf},
json={"confirmation": "DELETE"},
)
assert response.status_code == 200
assert corpus.deleted is True
def test_review_ui_uses_local_external_assets_and_keeps_guardrails_in_static_source() -> None:
web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web"
template = (web_root / "templates" / "index.html").read_text(encoding="utf-8")
script = (web_root / "static" / "review.js").read_text(encoding="utf-8")
stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8")
assert 'href="/static/review.css"' in template
assert 'src="/static/review.js"' in template
assert "<style" not in template
assert "<script>" not in template
assert "http://" not in template + script + stylesheet
assert "https://" not in template + script + stylesheet
assert "innerHTML" not in script
assert "Origin" not in script
assert (
'history.replaceState(null, "", `${window.location.pathname}${window.location.search}`)'
in script
)
assert '"/api/session/start"' in script
assert "JSON.stringify({ confirmation })" in script
assert "confidence: 1.0" in script
assert 'credentials: "same-origin"' in script
assert "localStorage" not in script
assert "sessionStorage" not in script
for expected in (
"weapon",
"gear",
"skill_chip",
"os_protocol",
"specialization",
"unknown",
"Mask",
"Gloves",
"Backpack",
"Body Armor",
"Holster",
"Knee Pads",
"prefers-reduced-motion",
):
assert expected in script + stylesheet
def test_review_page_and_local_static_assets_serve_with_security_headers() -> None:
client = TestClient(
create_app(
config=WebConfig(ServerConfig()),
capture_control=CaptureControl(),
corpus=Corpus(),
bootstrap_token=new_bootstrap_token(),
),
base_url="http://127.0.0.1:8765",
)
page = client.get("/")
stylesheet = client.get("/static/review.css")
script = client.get("/static/review.js")
for response in (page, stylesheet, script):
assert response.status_code == 200
assert response.headers["content-security-policy"].startswith("default-src 'self'")
assert response.headers["referrer-policy"] == "no-referrer"
assert response.headers["x-content-type-options"] == "nosniff"
assert response.headers["x-frame-options"] == "DENY"
assert "http://" not in response.text
assert "https://" not in response.text
assert 'href="/static/review.css"' in page.text
assert 'src="/static/review.js"' in page.text
assert stylesheet.headers["content-type"].startswith("text/css")
assert "csrfPattern" in script.text
assert "setCheckedMembers" in script.text
assert "announce: false" in script.text
@@ -0,0 +1,42 @@
from __future__ import annotations
import subprocess
import sys
import time
from pathlib import Path
import pytest
from game_data_capture.ingest.clipboard import ClipboardIntake, WatchConfig
def test_shutdown_reaps_real_child_process_group(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
child: subprocess.Popen[bytes] | None = None
real_popen = subprocess.Popen
def fake_popen(args: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
nonlocal child
assert args[0] == "wl-paste" and kwargs["shell"] is False
child = real_popen(
[sys.executable, "-c", "import time; time.sleep(30)"],
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
return child
monkeypatch.setattr(subprocess, "Popen", fake_popen)
intake = ClipboardIntake(
WatchConfig((tmp_path / "spool").resolve(), 100, 1, "v1", 1, 0.01, 0.2)
)
intake.arm()
assert child is not None and child.poll() is None
intake.shutdown()
for _ in range(20):
if child.poll() is not None:
break
time.sleep(0.01)
assert child.poll() is not None
@@ -0,0 +1,90 @@
from __future__ import annotations
from pathlib import Path
def test_review_interface_marks_keyboard_modal_and_responsive_state_hooks() -> None:
web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web"
template = (web_root / "templates" / "index.html").read_text(encoding="utf-8")
script = (web_root / "static" / "review.js").read_text(encoding="utf-8")
stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8")
assert 'aria-live="polite"' in template
assert '<dialog class="confirmation-dialog"' in template
assert 'value="cancel"' in template
assert 'type="radio"' not in template
assert "input.type = type" in script
assert 'input.type = "button"' not in script
assert 'input.type = "radio"' not in script
assert 'input.type = "checkbox"' not in script
assert "input.type = type" in script
assert "els.dialog.showModal()" in script
assert 'els.dialog.returnValue = "cancel"' in script
assert 'els.dialog.dataset.action = ""' in script
assert "const focusTarget = returnFocus;" in script
assert "if (focusTarget) focusTarget.focus();" in script
assert "fallbackFocus?.focus();" in script
assert 'els.media.addEventListener("error"' in script
assert ":focus-visible" in stylesheet
assert "min-height: 44px" in stylesheet
assert "@media (max-width: 52rem)" in stylesheet
assert "@media (prefers-reduced-motion: reduce)" in stylesheet
assert "@media (forced-colors: active)" in stylesheet
def test_review_script_guards_session_taxonomy_pending_and_selection_state() -> None:
script = (
Path(__file__).parents[2] / "src" / "game_data_capture" / "web" / "static" / "review.js"
).read_text(encoding="utf-8")
assert 'headers.set("Origin"' not in script
assert "Origin:" not in script
assert 'const bootstrap = await request("/api/bootstrap"' in script
assert 'const session = await request("/api/session")' in script
assert script.index('"/api/bootstrap"') < script.index('"/api/session"')
assert "if (!validSession(session))" in script
assert "family, false, () =>" in script
assert "view, false, renderDependentFields" in script
assert "if (!classification) { renderViews(); renderDependentFields(); return; }" in script
assert "function routeForAction(action)" in script
assert '(action === "reject" || action === "delete") && activeCapture' in script
assert "const routes =" not in script
assert 'document.querySelectorAll("#label-form input")' in script
assert "input.disabled = pending || locked || !activeCapture" in script
assert 'document.querySelectorAll("#capture-list button")' in script
assert "button.disabled = pending || locked;" in script
assert "armed === true" in script
assert "armed !== true" in script
assert "function acceptMutationCsrf(payload)" in script
assert "if (!payload || !acceptMutationCsrf(payload))" in script
assert 'retained ? "RETAINED"' in script
assert "function setCheckedMembers(name, values)" in script
assert "input.checked = selectedValues.has(input.value);" in script
assert 'setCheckedMembers("record_type", classification.record_types || []);' in script
assert 'forEach((record) => setChecked("record_type", record))' not in script
assert (
"async function refreshCaptures({ announce = true, fallbackFocus = null } = {})" in script
)
assert "async function selectCapture(captureId, { announce = true } = {})" in script
assert 'if (announce) notice("success", "QUEUE REFRESHED"' in script
assert 'if (announce) notice("success", "CAPTURE READY"' in script
assert "refreshCaptures({ announce: false })" in script
assert "selectCapture(captureId, { announce: false })" in script
assert script.index("selectCapture(captureId, { announce: false })") < script.index(
'notice("success", "LABEL SAVED"'
)
assert 'if (!activeCapture) els.reviewState.textContent = "READY";' in script
assert '"Capture is ARMED. Stop Session requires explicit confirmation."' in script
assert '"Capture is DISARMED. Start Session requires explicit confirmation."' in script
assert 'action === "start" ? els.stop' in script
assert 'action === "stop" ? els.start' in script
assert 'action === "delete" ? els.refresh' in script
assert "function focusIfAvailable(control)" in script
assert "focusIfAvailable(successFocus);" in script
assert "async function confirmed(action, fallbackFocus)" in script
assert "fallbackFocus?.focus();" in script
assert "function clearActiveDetail()" in script
assert 'if (action === "delete") clearActiveDetail();' in script
assert "captures.some((capture) => capture.capture_id === captureId)" in script
assert "refreshCaptures({ fallbackFocus: els.refresh })" in script
assert "focusIfAvailable(els.save);" in script
@@ -0,0 +1,496 @@
from __future__ import annotations
import io
import subprocess
from collections import deque
from collections.abc import Iterable, Mapping
from contextlib import AbstractContextManager, nullcontext
from pathlib import Path
from threading import Event
import pytest
import uvicorn
import game_data_capture.cli as cli
import game_data_capture.runtime as runtime_module
from game_data_capture.config import AppConfig, LifecycleConfig, ServerConfig
from game_data_capture.ingest.clipboard import (
ClipboardIntake,
FocusedWindowMetadata,
IntakeReceipt,
WatchConfig,
)
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, receipt_name, spool_png
from game_data_capture.paths import RuntimePaths
from game_data_capture.runtime import CaptureRuntime, InstanceLock, RuntimeFailure, _CaptureControl
from game_data_capture.storage import CaptureRepository, ContentAddressedBlobStore, StorageDatabase
_PNG = PNG_SIGNATURE + b"watch-png"
class FakeIntake:
def __init__(self, cancelled: Event, payloads: list[dict[str, object]]) -> None:
self.cancelled = cancelled
self.payloads = deque(payloads)
self.receipts: deque[IntakeReceipt] = deque()
self.armed = self.reconciled = self.shutdown_called = False
self.opened = 0
self.acknowledged: list[str] = []
self.seen_tokens: set[str] = set()
self._ticks = 0
def reconcile(self) -> list[str]:
self.reconciled = True
return []
def arm(self) -> None:
self.armed = True
def tick(self, now: float | None = None) -> None:
self._ticks += 1
if self._ticks > 1:
self.cancelled.set()
def take_helper_receipt(self) -> dict[str, object] | None:
return self.payloads.popleft() if self.payloads else None
def accept_helper_receipt(
self,
payload: Mapping[str, object],
*,
occurrence_id: str,
processing_version: str,
) -> bool:
token = str(payload["receipt_token"])
if token in self.seen_tokens:
return False
self.seen_tokens.add(token)
self.receipts.append(
IntakeReceipt(
occurrence_id,
token,
"0" * 64,
len(_PNG),
"image/png",
Path("/private/spool.png"),
"capture-v1",
processing_version,
FocusedWindowMetadata(None, None, None),
)
)
return True
def take(self) -> IntakeReceipt | None:
return self.receipts.popleft() if self.receipts else None
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]:
self.opened += 1
return nullcontext([_PNG])
def acknowledge(self, receipt: IntakeReceipt) -> bool:
self.acknowledged.append(receipt.receipt_token)
return True
def shutdown(self) -> None:
self.shutdown_called = True
def _runtime(tmp_path: Path, intake: FakeIntake) -> CaptureRuntime:
root = tmp_path / "xdg"
paths = RuntimePaths(
config_dir=(root / "config").resolve(),
data_dir=(root / "data").resolve(),
cache_dir=(root / "cache").resolve(),
runtime_dir=(root / "runtime").resolve(),
)
return CaptureRuntime(
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
paths,
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
intake_factory=lambda _config: intake,
)
def _payload(token: str) -> dict[str, object]:
return {
"receipt_version": 1,
"receipt_token": token,
"sha256": "0" * 64,
"byte_size": len(_PNG),
"media_type": "image/png",
"spool_name": "ignored.png",
"policy_version": "capture-v1",
}
class PassiveClipboardIntake(ClipboardIntake):
"""Use real private spool sidecars without launching a Wayland watcher."""
def __init__(
self, config: WatchConfig, cancelled: Event, *, acknowledge_result: bool = True
) -> None:
super().__init__(config)
self.cancelled = cancelled
self.acknowledge_result = acknowledge_result
self.acknowledged: list[str] = []
self.reconcile_calls = 0
self._ticks = 0
def reconcile(self) -> list[str]:
self.reconcile_calls += 1
return super().reconcile()
def arm(self) -> None:
self.armed, self.paused = True, False
def tick(self, now: float | None = None) -> None:
self._ticks += 1
if self._ticks > 1:
self.cancelled.set()
def acknowledge(self, receipt: IntakeReceipt) -> bool:
self.acknowledged.append(receipt.receipt_token)
return self.acknowledge_result and super().acknowledge(receipt)
class StopTimeoutReceiptIntake(PassiveClipboardIntake):
def __init__(self, config: WatchConfig, cancelled: Event) -> None:
super().__init__(config, cancelled)
self.tick_entered = Event()
self.release_tick = Event()
def tick(self, now: float | None = None) -> None:
self.tick_entered.set()
self.release_tick.wait()
def shutdown(self) -> None:
self.release_tick.set()
super().shutdown()
def _durable_runtime(
tmp_path: Path, cancelled: Event, *, acknowledge_result: bool = True
) -> tuple[CaptureRuntime, list[PassiveClipboardIntake]]:
created: list[PassiveClipboardIntake] = []
root = tmp_path / "durable-xdg"
paths = RuntimePaths(
config_dir=(root / "config").resolve(),
data_dir=(root / "data").resolve(),
cache_dir=(root / "cache").resolve(),
runtime_dir=(root / "runtime").resolve(),
)
def factory(config: WatchConfig) -> PassiveClipboardIntake:
intake = PassiveClipboardIntake(config, cancelled, acknowledge_result=acknowledge_result)
created.append(intake)
return intake
return (
CaptureRuntime(
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
paths,
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
intake_factory=factory,
),
created,
)
def _durable_receipt(runtime: CaptureRuntime, token: str) -> None:
spool_png(
io.BytesIO(_PNG),
spool=runtime.paths.cache_dir / "spool",
max_bytes=100,
receipt_token=token,
policy_version="capture-v1",
)
def test_watch_streams_verified_receipts_and_replays_exact_tokens_idempotently(
tmp_path: Path,
) -> None:
cancelled = Event()
intake = FakeIntake(cancelled, [_payload("watch-receipt-1"), _payload("watch-receipt-1")])
runtime = _runtime(tmp_path, intake)
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir)
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
assert intake.armed and intake.reconciled and intake.shutdown_called and intake.opened == 1
assert intake.acknowledged == ["watch-receipt-1"]
def test_watch_ignores_an_absent_helper_receipt_without_error_noise(tmp_path: Path) -> None:
cancelled = Event()
intake = FakeIntake(cancelled, [])
runtime = _runtime(tmp_path, intake)
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir)
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
assert intake.shutdown_called and intake.opened == 0
def test_startup_reconciles_partial_blob_storage_before_arming(tmp_path: Path) -> None:
cancelled = Event()
intake = FakeIntake(cancelled, [])
runtime = _runtime(tmp_path, intake)
blobs = ContentAddressedBlobStore(runtime.paths.data_dir)
blobs.temporary.mkdir(parents=True)
partial = blobs.temporary / ".leftover.partial"
partial.write_bytes(b"partial")
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
assert not partial.exists()
assert intake.reconciled and intake.shutdown_called
def test_durable_receipt_is_acknowledged_once_after_repository_confirmation(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cancelled = Event()
runtime, created = _durable_runtime(tmp_path, cancelled)
_durable_receipt(runtime, "confirmed")
events: list[str] = []
original_receive = CaptureRepository.receive
original_acknowledge = PassiveClipboardIntake.acknowledge
def tracked_receive(
self: CaptureRepository,
capture_id: str,
receipt_token: str,
chunks: Iterable[bytes],
*,
media_type: str,
max_bytes: int,
) -> str:
events.append("receive")
return original_receive(
self,
capture_id,
receipt_token,
chunks,
media_type=media_type,
max_bytes=max_bytes,
)
monkeypatch.setattr(CaptureRepository, "receive", tracked_receive)
def tracked_acknowledge(self: PassiveClipboardIntake, receipt: IntakeReceipt) -> bool:
events.append("acknowledge")
return original_acknowledge(self, receipt)
monkeypatch.setattr(PassiveClipboardIntake, "acknowledge", tracked_acknowledge)
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
assert events == ["receive", "acknowledge"]
assert created[0].acknowledged == ["confirmed"]
assert created[0].reconcile_calls == 1
assert not (runtime.paths.cache_dir / "spool" / receipt_name("confirmed")).exists()
def test_storage_failure_leaves_durable_receipt_unacknowledged(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cancelled = Event()
runtime, created = _durable_runtime(tmp_path, cancelled)
_durable_receipt(runtime, "storage-failure")
def failed_receive(
self: CaptureRepository,
capture_id: str,
receipt_token: str,
chunks: Iterable[bytes],
*,
media_type: str,
max_bytes: int,
) -> str:
raise OSError("storage unavailable")
monkeypatch.setattr(CaptureRepository, "receive", failed_receive)
with pytest.raises(RuntimeFailure, match="capture watch failed"):
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
assert created[0].acknowledged == []
assert (runtime.paths.cache_dir / "spool" / receipt_name("storage-failure")).exists()
def test_cancellation_after_storage_confirmation_does_not_acknowledge(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cancelled = Event()
runtime, created = _durable_runtime(tmp_path, cancelled)
_durable_receipt(runtime, "cancelled-after-storage")
original_receive = CaptureRepository.receive
def cancel_after_receive(
self: CaptureRepository,
capture_id: str,
receipt_token: str,
chunks: Iterable[bytes],
*,
media_type: str,
max_bytes: int,
) -> str:
result = original_receive(
self,
capture_id,
receipt_token,
chunks,
media_type=media_type,
max_bytes=max_bytes,
)
cancelled.set()
return result
monkeypatch.setattr(CaptureRepository, "receive", cancel_after_receive)
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
assert created[0].acknowledged == []
assert (runtime.paths.cache_dir / "spool" / receipt_name("cancelled-after-storage")).exists()
def test_acknowledgement_failure_replays_idempotently_and_releases_lock(tmp_path: Path) -> None:
cancelled = Event()
runtime, failed = _durable_runtime(tmp_path, cancelled, acknowledge_result=False)
_durable_receipt(runtime, "replay-token")
with pytest.raises(RuntimeFailure, match="acknowledgement failed"):
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
assert failed[0].acknowledged == ["replay-token"]
sidecar = runtime.paths.cache_dir / "spool" / receipt_name("replay-token")
assert sidecar.exists()
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
reusable.release()
replay_cancelled = Event()
replay_runtime, replayed = _durable_runtime(tmp_path, replay_cancelled)
replay_runtime.watch(cancelled=replay_cancelled, poll_seconds=0.001)
database = StorageDatabase(replay_runtime.paths.data_dir, replay_runtime.migrations_dir)
with database.connect() as connection:
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
assert replayed[0].acknowledged == ["replay-token"]
assert not sidecar.exists()
def test_service_stop_timeout_preserves_unacknowledged_durable_sidecar(tmp_path: Path) -> None:
cancelled = Event()
created: list[StopTimeoutReceiptIntake] = []
root = tmp_path / "stop-timeout-xdg"
paths = RuntimePaths(
config_dir=(root / "config").resolve(),
data_dir=(root / "data").resolve(),
cache_dir=(root / "cache").resolve(),
runtime_dir=(root / "runtime").resolve(),
)
def factory(config: WatchConfig) -> StopTimeoutReceiptIntake:
intake = StopTimeoutReceiptIntake(config, cancelled)
created.append(intake)
return intake
runtime = CaptureRuntime(
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0, shutdown_timeout_seconds=1)),
paths,
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
intake_factory=factory,
)
_durable_receipt(runtime, "timeout-receipt")
resources = runtime._start()
control = _CaptureControl(runtime, resources.repository)
try:
assert control.start_session() == "started"
assert created[0].tick_entered.wait(0.2)
assert control.stop_session() == "stopped"
assert created[0].acknowledged == []
assert (runtime.paths.cache_dir / "spool" / receipt_name("timeout-receipt")).exists()
finally:
control.shutdown()
resources.close()
def test_cli_bare_watch_is_a_safe_refusal_and_modes_never_start_live_processes(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
root = tmp_path / "fresh-xdg"
for key, name in (
("HOME", "home"),
("XDG_CONFIG_HOME", "config"),
("XDG_DATA_HOME", "data"),
("XDG_CACHE_HOME", "cache"),
("XDG_RUNTIME_DIR", "runtime"),
):
monkeypatch.setenv(key, str(root / name))
def forbidden(*_arguments: object, **_kwargs: object) -> None:
raise AssertionError("test started a process")
monkeypatch.setattr(subprocess, "Popen", forbidden)
assert cli.main(["watch"]) == cli.EXIT_UNIMPLEMENTED
assert "requires gdc-runtime-integration" in capsys.readouterr().err
assert cli.main(["watch", "--dry-run"]) == 0
assert capsys.readouterr().out == "watch dry-run: capture runtime plan is valid\n"
assert not root.exists()
def test_cli_arm_reaches_only_a_mocked_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[str] = []
class FakeRuntime:
def __init__(self, _config: object, _paths: object) -> None:
calls.append("created")
def watch(self) -> None:
calls.append("watch")
monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime)
assert cli.main(["watch", "--arm"]) == 0
assert calls == ["created", "watch"]
def test_cli_serve_validates_before_constructing_resources_and_passes_exact_authority(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[tuple[str, object]] = []
class FakeRuntime:
def __init__(self, config: AppConfig, _paths: object) -> None:
calls.append(("created", config.server))
def serve(self) -> None:
calls.append(("serve", None))
monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime)
assert cli.main(["serve", "--host", "127.0.0.1", "--port", "9877"]) == 0
assert calls == [("created", ServerConfig("127.0.0.1", 9877)), ("serve", None)]
calls.clear()
assert cli.main(["serve", "--host", "0.0.0.0"]) == cli.EXIT_CONFIG_ERROR
assert calls == []
def test_cli_serve_converts_uvicorn_startup_exit_without_leaking_its_token(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
root = tmp_path / "serve-xdg"
for key, name in (
("HOME", "home"),
("XDG_CONFIG_HOME", "config"),
("XDG_DATA_HOME", "data"),
("XDG_CACHE_HOME", "cache"),
("XDG_RUNTIME_DIR", "runtime"),
):
monkeypatch.setenv(key, str(root / name))
token = "Z" * 43
monkeypatch.setattr(runtime_module, "new_bootstrap_token", lambda: token)
def failed_server(*_: object, **__: object) -> None:
raise SystemExit(2)
monkeypatch.setattr(uvicorn, "run", failed_server)
assert cli.main(["serve"]) == cli.EXIT_RUNTIME_ERROR
output = capsys.readouterr()
assert output.err == "capture runtime failed\n"
assert token not in output.err
assert "SystemExit" not in output.err
reusable = InstanceLock.acquire(root / "runtime" / "shd-capture" / "instance.lock", 0)
reusable.release()
@@ -0,0 +1,74 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from fastapi.testclient import TestClient
from game_data_capture.config import ServerConfig
from game_data_capture.web.app import (
CaptureRecord,
CaptureSessionStatus,
ControlOutcome,
MediaDescriptor,
MutationOutcome,
WebConfig,
create_app,
)
from game_data_capture.web.security import new_bootstrap_token
class Control:
def status(self) -> CaptureSessionStatus:
return CaptureSessionStatus(False, 0, 1)
def start_session(self) -> ControlOutcome:
return "started"
def stop_session(self) -> ControlOutcome:
return "stopped"
class EmptyCorpus:
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
return ()
def get_capture(self, capture_id: str) -> CaptureRecord | None:
return None
def open_media(self, capture_id: str) -> MediaDescriptor | None:
return None
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
return "not_found"
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
return "not_found"
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
return "not_found"
def test_bootstrap_is_one_time_same_origin_and_does_not_enable_permissive_cors() -> None:
token = new_bootstrap_token()
app = create_app(
config=WebConfig(ServerConfig()),
capture_control=Control(),
corpus=EmptyCorpus(),
bootstrap_token=token,
)
client = TestClient(app, base_url="http://127.0.0.1:8765")
cross_origin = client.post(
"/api/bootstrap", headers={"Origin": "http://evil.test"}, json={"token": token}
)
accepted = client.post(
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
)
replay = client.post(
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
)
assert cross_origin.status_code == 403
assert accepted.status_code == 200
assert replay.status_code == 401
assert "access-control-allow-origin" not in accepted.headers