Add game data capture vertical slice
This commit is contained in:
@@ -0,0 +1,458 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.storage import CaptureCorpus, CaptureRepository, ContentAddressedBlobStore
|
||||
from game_data_capture.storage.blobs import BlobCorruptionError, BlobLockError
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import StorageError
|
||||
|
||||
|
||||
def _label(*, view: str = "detail", records: list[str] | None = None) -> dict[str, object]:
|
||||
return {
|
||||
"family": "weapon",
|
||||
"view": view,
|
||||
"gear_slot": None,
|
||||
"record_types": ["weapon"] if records is None else records,
|
||||
"confidence": 1.0,
|
||||
"abstention_reason": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corpus(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository, CaptureCorpus]:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
database = StorageDatabase(
|
||||
root,
|
||||
(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
)
|
||||
database.migrate()
|
||||
blobs = ContentAddressedBlobStore(root)
|
||||
repository = CaptureRepository(database, blobs)
|
||||
return database, repository, CaptureCorpus(database, blobs, max_media_bytes=1024)
|
||||
|
||||
|
||||
def _receive(
|
||||
repository: CaptureRepository, capture_id: str, token: str, payload: bytes = b"png"
|
||||
) -> None:
|
||||
repository.receive(capture_id, token, [payload], media_type="image/png", max_bytes=1024)
|
||||
|
||||
|
||||
def _assert_media_unavailable(port: CaptureCorpus, capture_id: str) -> None:
|
||||
media = port.open_media(capture_id)
|
||||
assert media is not None
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
next(iter(media.chunks))
|
||||
|
||||
|
||||
def _commit_deletion_intent(database: StorageDatabase, capture_id: str) -> None:
|
||||
with database.connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT c.blob_sha256, c.state, c.received_at, c.receipt_token, c.failure_reason, "
|
||||
"c.revision, b.byte_size FROM captures AS c JOIN blobs AS b "
|
||||
"ON b.sha256 = c.blob_sha256 WHERE c.capture_id = ?",
|
||||
(capture_id,),
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute(
|
||||
"INSERT INTO capture_deletion_intents(blob_sha256, capture_id, byte_size, "
|
||||
"capture_state, received_at, receipt_token, failure_reason, revision, created_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
row["blob_sha256"],
|
||||
capture_id,
|
||||
row["byte_size"],
|
||||
row["state"],
|
||||
row["received_at"],
|
||||
row["receipt_token"],
|
||||
row["failure_reason"],
|
||||
row["revision"],
|
||||
"time",
|
||||
),
|
||||
)
|
||||
connection.execute("DELETE FROM captures WHERE capture_id = ?", (capture_id,))
|
||||
connection.execute("COMMIT")
|
||||
|
||||
|
||||
def test_safe_bounded_listing_detail_and_verified_media(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
records = port.list_captures(limit=1)
|
||||
assert len(records) == 1 and records[0].capture_id == "capture-0002"
|
||||
assert port.get_capture("../capture-0001") is None
|
||||
assert port.open_media("../capture-0001") is None
|
||||
media = port.open_media("capture-0001")
|
||||
assert media is not None
|
||||
assert media.media_type == "image/png" and media.byte_size == 3
|
||||
chunks = media.chunks
|
||||
assert next(iter(chunks)) == b"png"
|
||||
close = getattr(chunks, "close", None)
|
||||
assert callable(close)
|
||||
close()
|
||||
for invalid in (0, 257, True):
|
||||
with pytest.raises(ValueError):
|
||||
port.list_captures(limit=invalid)
|
||||
|
||||
|
||||
def test_labels_preserve_prior_values_across_restart_and_reject_is_idempotent(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
first = _label()
|
||||
second = _label(view="blueprint", records=["weapon_talent"])
|
||||
with pytest.raises(ValueError):
|
||||
port.label("capture-0001", {"family": "weapon"})
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", second) == "updated"
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
record = restarted.get_capture("capture-0001")
|
||||
assert record is not None and record.classification == second
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
|
||||
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert restarted.label("capture-0001", first) == "conflict"
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_rejections").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_concurrent_label_writes_serialize_without_losing_prior_value(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
labels = (_label(), _label(view="blueprint", records=["weapon_talent"]))
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
outcomes = list(executor.map(lambda value: port.label("capture-0001", value), labels))
|
||||
assert outcomes == ["updated", "updated"]
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_canonical_equivalent_labels_do_not_append_history(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
first = _label(records=["weapon", "weapon_talent"])
|
||||
equivalent = _label(records=["weapon_talent", "weapon"])
|
||||
equivalent["confidence"] = 1
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", equivalent) == "updated"
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_rejected_or_evidence_bearing_captures_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
repository.append_ocr_run("run-0002", "capture-0002", "ocr", "1", {})
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
assert port.delete_with_retention("bad") == "not_found"
|
||||
|
||||
|
||||
def test_label_and_job_evidence_retain_capture(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.label("capture-0001", _label()) == "updated"
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO jobs(job_id, capture_id, kind, state, max_attempts) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
("job-0002", "capture-0002", "ocr", "queued", 1),
|
||||
)
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
|
||||
|
||||
def test_receive_translates_put_lock_failure_without_capture(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, _ = corpus
|
||||
|
||||
def locked(_: object, *, max_bytes: int) -> tuple[str, int]:
|
||||
raise BlobLockError("locked")
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "put", locked)
|
||||
with pytest.raises(StorageError, match="unavailable"):
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("state", "failure_reason"),
|
||||
(("failed", None), ("cancelled", None), ("stored", "capture failure"), ("unknown", None)),
|
||||
)
|
||||
def test_non_disposable_capture_rows_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
state: str,
|
||||
failure_reason: str | None,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"UPDATE captures SET state = ?, failure_reason = ? WHERE capture_id = ?",
|
||||
(state, failure_reason, "capture-0001"),
|
||||
)
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
|
||||
|
||||
def test_capture_lineage_and_failure_evidence_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
repository.append_lineage("lineage-0001", "capture-0001", "parent-0001", "derived")
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
repository.record_failure("failure-0002", "capture", "capture-0002", "fault", "bounded")
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
_receive(repository, "capture-0003", "receipt-0003")
|
||||
repository.append_lineage("lineage-0003", "child-0003", "capture-0003", "derived")
|
||||
assert port.delete_with_retention("capture-0003") == "retained"
|
||||
|
||||
|
||||
def test_shared_blob_deletion_keeps_remaining_capture_media(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001", b"shared")
|
||||
_receive(repository, "capture-0002", "receipt-0002", b"shared")
|
||||
assert port.delete_with_retention("capture-0001") == "updated"
|
||||
assert port.get_capture("capture-0001") is None
|
||||
media = port.open_media("capture-0002")
|
||||
assert media is not None and b"".join(media.chunks) == b"shared"
|
||||
with repository.database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_unshared_deletion_removes_media_metadata_and_intent(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
assert port.delete_with_retention("capture-0001") == "updated"
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
|
||||
|
||||
def test_committed_intent_before_unlink_blocks_replay_then_finalizes(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
_commit_deletion_intent(database, "capture-0001")
|
||||
assert path.exists()
|
||||
with pytest.raises(StorageError, match="deletion"):
|
||||
_receive(repository, "capture-0002", "receipt-0001")
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
assert restarted.reconcile_deletions() == 1
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
|
||||
|
||||
def test_interleaved_reinstall_waits_for_deletion_finalization(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001", b"shared")
|
||||
unlinked = threading.Event()
|
||||
release = threading.Event()
|
||||
original = repository.blobs.delete_verified
|
||||
outcomes: list[object] = []
|
||||
|
||||
def pause(digest: str, *, expected_size: int) -> bool:
|
||||
result = original(digest, expected_size=expected_size)
|
||||
unlinked.set()
|
||||
assert release.wait(0.5)
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "delete_verified", pause)
|
||||
deletion = threading.Thread(
|
||||
target=lambda: outcomes.append(port.delete_with_retention("capture-0001"))
|
||||
)
|
||||
deletion.start()
|
||||
assert unlinked.wait(0.5)
|
||||
receipt = threading.Thread(
|
||||
target=lambda: outcomes.append(
|
||||
repository.receive(
|
||||
"capture-0002", "receipt-0002", [b"shared"], media_type="image/png", max_bytes=1024
|
||||
)
|
||||
)
|
||||
)
|
||||
receipt.start()
|
||||
release.set()
|
||||
deletion.join(1)
|
||||
receipt.join(1)
|
||||
assert not deletion.is_alive() and not receipt.is_alive()
|
||||
assert set(outcomes) == {"updated", "capture-0002"}
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
media = port.open_media("capture-0002")
|
||||
assert media is not None and b"".join(media.chunks) == b"shared"
|
||||
|
||||
|
||||
def test_unlink_then_sqlite_finalization_failure_recovers(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT sha256, byte_size FROM blobs").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["sha256"]))
|
||||
_commit_deletion_intent(database, "capture-0001")
|
||||
repository.blobs.delete_verified(str(row["sha256"]), expected_size=int(row["byte_size"]))
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"CREATE TRIGGER fail_finalization BEFORE DELETE ON capture_deletion_intents "
|
||||
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
|
||||
)
|
||||
with pytest.raises(sqlite3.IntegrityError, match="fault"):
|
||||
port.reconcile_deletions()
|
||||
assert port.get_capture("capture-0001") is None
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 1
|
||||
)
|
||||
connection.execute("DROP TRIGGER fail_finalization")
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
assert restarted.reconcile_deletions() == 1
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_corrupt_missing_and_symlink_media_fail_closed(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], tmp_path: Path
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with repository.database.connect() as connection:
|
||||
row = connection.execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
path.write_bytes(b"x" * 2048)
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
path.unlink()
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
path.symlink_to(tmp_path / "outside")
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
|
||||
|
||||
def test_label_transaction_rolls_back_on_evidence_failure(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.label("capture-0001", _label()) == "updated"
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"CREATE TRIGGER fail_label_evidence BEFORE INSERT ON capture_label_evidence "
|
||||
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
|
||||
)
|
||||
with pytest.raises(sqlite3.IntegrityError, match="fault"):
|
||||
port.label(
|
||||
"capture-0001",
|
||||
_label(view="blueprint", records=["weapon_talent"]),
|
||||
)
|
||||
assert port.get_capture("capture-0001") is not None
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
|
||||
stored = connection.execute("SELECT classification_json FROM capture_labels").fetchone()
|
||||
assert stored is not None and '"view":"detail"' in stored["classification_json"]
|
||||
|
||||
|
||||
def test_deletion_intent_recovers_after_unlink_failure(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
original = tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
def fail_unlink(_: str, *, expected_size: int) -> bool:
|
||||
raise BlobCorruptionError("injected fault")
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "delete_verified", fail_unlink)
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert (
|
||||
tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
)
|
||||
== original
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
media = port.open_media("capture-0001")
|
||||
assert media is not None and b"".join(media.chunks) == b"png"
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.storage.database import MigrationError, StorageDatabase
|
||||
|
||||
|
||||
def _migrations(tmp_path: Path) -> Path:
|
||||
source = Path(__file__).parents[3] / "migrations" / "001_initial.sql"
|
||||
destination = tmp_path / "migrations"
|
||||
destination.mkdir()
|
||||
shutil.copy(source, destination / source.name)
|
||||
return destination
|
||||
|
||||
|
||||
def test_connection_pragmas_and_private_modes(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
database = StorageDatabase(root, _migrations(tmp_path).resolve())
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS live_wal(value INTEGER)")
|
||||
assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1
|
||||
assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||||
assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000
|
||||
assert connection.execute("PRAGMA synchronous").fetchone()[0] == 2
|
||||
for path in (
|
||||
database.path.with_name("capture.sqlite3-wal"),
|
||||
database.path.with_name("capture.sqlite3-shm"),
|
||||
):
|
||||
if path.exists():
|
||||
assert path.stat().st_mode & 0o077 == 0
|
||||
connection.execute("COMMIT")
|
||||
assert root.stat().st_mode & 0o777 == 0o700
|
||||
assert database.path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_database_symlink_is_rejected(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
root.mkdir()
|
||||
outside = tmp_path / "outside"
|
||||
outside.write_text("x")
|
||||
(root / "capture.sqlite3").symlink_to(outside)
|
||||
with pytest.raises(MigrationError):
|
||||
StorageDatabase(root, _migrations(tmp_path).resolve()).connect()
|
||||
|
||||
|
||||
def test_migration_history_and_failure_rollback(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
first = migrations / "001_initial.sql"
|
||||
first.write_text(first.read_text() + "\n-- changed\n")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
first.write_text((Path(__file__).parents[3] / "migrations" / "001_initial.sql").read_text())
|
||||
bad = migrations / "002_bad.sql"
|
||||
bad.write_text("CREATE TABLE partial_test(value INTEGER); BAD SQL;")
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE name='partial_test'"
|
||||
).fetchone()
|
||||
is None
|
||||
)
|
||||
bad.write_text("CREATE TABLE partial_test(value INTEGER);")
|
||||
database.migrate()
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
|
||||
|
||||
|
||||
def test_migration_removal_rename_and_duplicate_rejected(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
path = migrations / "001_initial.sql"
|
||||
path.rename(migrations / "001_renamed.sql")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
(migrations / "001_renamed.sql").rename(path)
|
||||
path.unlink()
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
shutil.copy(Path(__file__).parents[3] / "migrations" / "001_initial.sql", path)
|
||||
shutil.copy(path, migrations / "001_duplicate.sql")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
|
||||
|
||||
def test_corpus_migration_preserves_populated_initial_database(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO blobs(sha256, byte_size, media_type, created_at) VALUES (?, 1, ?, ?)",
|
||||
("0" * 64, "image/png", "time"),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO captures(capture_id, blob_sha256, state, received_at, receipt_token) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
("capture-0001", "0" * 64, "stored", "time", "receipt-0001"),
|
||||
)
|
||||
source = Path(__file__).parents[3] / "migrations" / "002_capture_corpus.sql"
|
||||
shutil.copy(source, migrations / source.name)
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 0
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT sha256, byte_size, media_type, created_at FROM blobs"
|
||||
).fetchone()
|
||||
) == ("0" * 64, 1, "image/png", "time")
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
) == ("capture-0001", "0" * 64, "stored", "time", "receipt-0001", None, 0)
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
|
||||
migration = connection.execute(
|
||||
"SELECT filename, checksum FROM schema_migrations WHERE version = 2"
|
||||
).fetchone()
|
||||
assert migration is not None
|
||||
assert migration["filename"] == "002_capture_corpus.sql"
|
||||
assert isinstance(migration["checksum"], str) and len(migration["checksum"]) == 64
|
||||
names = {row["name"] for row in connection.execute("SELECT name FROM sqlite_master")}
|
||||
assert {
|
||||
"captures_blob_sha256",
|
||||
"jobs_capture_id",
|
||||
"ocr_runs_capture_id",
|
||||
"runtime_failures_capture_subject",
|
||||
"lineage_child_id",
|
||||
"lineage_parent_id",
|
||||
}.issubset(names)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.pipeline.jobs import JobLeases
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import StorageError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database(tmp_path: Path) -> StorageDatabase:
|
||||
database = StorageDatabase(
|
||||
(tmp_path / "utility").resolve(), (Path(__file__).parents[3] / "migrations").resolve()
|
||||
)
|
||||
database.migrate()
|
||||
return database
|
||||
|
||||
|
||||
def _expire(database: StorageDatabase, job_id: str) -> None:
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"UPDATE jobs SET lease_expires_at = ? WHERE job_id = ?",
|
||||
((datetime.now(UTC) - timedelta(seconds=1)).isoformat(), job_id),
|
||||
)
|
||||
|
||||
|
||||
def test_job_validation_and_exclusive_ownership(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
for job_id, kind, attempts in (
|
||||
("!", "ocr", 1),
|
||||
("job", "bad", 1),
|
||||
("job", "ocr", True),
|
||||
("job", "ocr", 0),
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.create(job_id, kind, max_attempts=attempts)
|
||||
jobs.create("job-1", "ocr")
|
||||
for duration in (0, 301, True):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.acquire("owner-1", duration)
|
||||
with pytest.raises(ValueError):
|
||||
jobs.acquire("!", 30)
|
||||
assert jobs.acquire("owner-1", 30) == "job-1"
|
||||
assert jobs.acquire("owner-2", 30) is None
|
||||
with pytest.raises(StorageError):
|
||||
jobs.renew("job-1", "owner-2", 30)
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("job-1", "owner-2", "succeeded")
|
||||
for duration in (0, 301, True):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.renew("job-1", "owner-1", duration)
|
||||
|
||||
|
||||
def test_expiry_recovery_exhaustion_and_terminal_jobs(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("retry", "ocr", max_attempts=2)
|
||||
assert jobs.acquire("owner-1", 30) == "retry"
|
||||
_expire(database, "retry")
|
||||
with pytest.raises(StorageError):
|
||||
jobs.renew("retry", "owner-1", 30)
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("retry", "owner-1", "succeeded")
|
||||
assert jobs.recover() == 1
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id='retry'"
|
||||
).fetchone()
|
||||
) == ("queued", None, None)
|
||||
assert JobLeases(database).acquire("owner-2", 30) == "retry"
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT attempt, state, lease_owner FROM jobs WHERE job_id='retry'"
|
||||
).fetchone()
|
||||
) == (2, "running", "owner-2")
|
||||
jobs.create("exhaust", "ocr", max_attempts=1)
|
||||
assert jobs.acquire("owner-3", 30) == "exhaust"
|
||||
_expire(database, "exhaust")
|
||||
assert jobs.recover() == 1
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, attempt, lease_owner, lease_expires_at "
|
||||
"FROM jobs WHERE job_id='exhaust'"
|
||||
).fetchone()
|
||||
) == ("failed", 1, None, None)
|
||||
assert jobs.acquire("owner-4", 30) is None
|
||||
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
|
||||
jobs.create(job_id, "ocr")
|
||||
assert jobs.acquire("owner-5", 30) == job_id
|
||||
jobs.finish(job_id, "owner-5", target)
|
||||
with database.connect() as connection:
|
||||
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id = ?",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
) == (target, None, None)
|
||||
|
||||
|
||||
def test_retry_wait_finish_rejected_without_changing_lease(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("job-1", "ocr")
|
||||
jobs.acquire("owner-1", 30)
|
||||
with pytest.raises(ValueError):
|
||||
jobs.finish("job-1", "owner-1", "retry_wait")
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner FROM jobs WHERE job_id='job-1'"
|
||||
).fetchone()
|
||||
) == ("running", "owner-1")
|
||||
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.pipeline.jobs import JobLeases
|
||||
from game_data_capture.storage.blobs import ContentAddressedBlobStore
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import CaptureRepository, StorageError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository]:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = Path(__file__).parents[3] / "migrations"
|
||||
database = StorageDatabase(root, migrations.resolve())
|
||||
database.migrate()
|
||||
return database, CaptureRepository(database, ContentAddressedBlobStore(root))
|
||||
|
||||
|
||||
def test_identical_occurrences_are_deduplicated_and_receipts_survive_restart(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-1"
|
||||
)
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-2", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-2"
|
||||
)
|
||||
# A simulated restart uses fresh repository objects and a repeated intake receipt.
|
||||
restarted = CaptureRepository(database, ContentAddressedBlobStore(database.root))
|
||||
assert (
|
||||
restarted.receive(
|
||||
"capture-other", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-2"
|
||||
)
|
||||
assert restarted.report()["blobs"] == 1
|
||||
assert restarted.report()["captures"] == 2
|
||||
|
||||
|
||||
def test_capture_never_references_missing_blob_and_cas_rejects_stale(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"complete"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None and repository.blobs.verify(str(row["blob_sha256"])) == len(b"complete")
|
||||
assert repository.transition("capture-1", "queued", 0) == 1
|
||||
with pytest.raises(StorageError):
|
||||
repository.transition("capture-1", "failed", 0)
|
||||
|
||||
|
||||
def test_reprocessing_is_append_only_and_leases_enforce_owner(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {"raw": "one"})
|
||||
repository.append_ocr_run("run-2", "capture-1", "ocr", "2", {"raw": "two"})
|
||||
assert database.connect().execute("SELECT count(*) FROM ocr_runs").fetchone()[0] == 2
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("job-1", "ocr", "capture-1")
|
||||
assert jobs.acquire("worker-a", 30) == "job-1"
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("job-1", "worker-b", "succeeded")
|
||||
jobs.finish("job-1", "worker-a", "succeeded")
|
||||
|
||||
|
||||
def test_replayed_receipt_does_not_consume_payload(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
|
||||
consumed = [False]
|
||||
|
||||
class MustNotConsume:
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
consumed[0] = True
|
||||
raise AssertionError("receipt replay consumed payload")
|
||||
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-2", "receipt-1", MustNotConsume(), media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-1"
|
||||
)
|
||||
assert not consumed[0]
|
||||
|
||||
|
||||
def test_every_provenance_kind_is_insert_only(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
|
||||
repository.append_region("region-1", "run-1", {})
|
||||
repository.append_classification("class-1", "run-1", {})
|
||||
repository.append_candidate("candidate-1", "run-1", {})
|
||||
repository.append_routing("route-1", "candidate-1", {})
|
||||
repository.append_review("review-1", "candidate-1", {})
|
||||
repository.append_lineage("lineage-1", "child-1", "parent-1", "derived")
|
||||
repository.append_export("export-1", "ready", {})
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
|
||||
payloads = {
|
||||
"ocr_runs": "payload_json",
|
||||
"regions": "region_json",
|
||||
"classifications": "classification_json",
|
||||
"extraction_candidates": "candidate_json",
|
||||
"routing_decisions": "decision_json",
|
||||
"review_events": "event_json",
|
||||
"exports": "manifest_json",
|
||||
}
|
||||
with database.connect() as connection:
|
||||
for table in (
|
||||
"ocr_runs",
|
||||
"regions",
|
||||
"classifications",
|
||||
"extraction_candidates",
|
||||
"routing_decisions",
|
||||
"review_events",
|
||||
"lineage",
|
||||
"exports",
|
||||
):
|
||||
assert connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0] == 1
|
||||
for table, column in payloads.items():
|
||||
json.loads(connection.execute(f"SELECT {column} FROM {table}").fetchone()[0])
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM ocr_runs WHERE run_id='run-1'").fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_failure_sanitization_and_provenance_bounds(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_ocr_run("run", "capture-1", "", "1", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_ocr_run("run", "capture-1", "x" * 101, "1", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_lineage("line", "child", "parent", "")
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_export("export", "", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_candidate("candidate", "run", "x" * 1_000_001)
|
||||
long_token = "x" * 600
|
||||
short_message = (
|
||||
"sk-secret123 Authorization: Bearer auth-secret token=token-secret secret=hidden "
|
||||
"password=pw-secret api_key=key-secret /home/agent/private.png\n"
|
||||
)
|
||||
repository.record_failure("failure-1", "capture", "capture-1", "failure-code", short_message)
|
||||
repository.record_failure("failure-2", "capture", "capture-1", "failure-code", long_token)
|
||||
with database.connect() as connection:
|
||||
rows = list(
|
||||
connection.execute("SELECT code, message FROM runtime_failures ORDER BY failure_id")
|
||||
)
|
||||
assert len(rows) == 2
|
||||
for row in rows:
|
||||
assert len(row["message"]) <= 512 and len(row["code"]) <= 64
|
||||
assert "\n" not in row["message"]
|
||||
for value in (
|
||||
"sk-secret123",
|
||||
"auth-secret",
|
||||
"token-secret",
|
||||
"hidden",
|
||||
"pw-secret",
|
||||
"key-secret",
|
||||
"/home/agent/private.png",
|
||||
"\n",
|
||||
):
|
||||
assert value not in rows[0]["message"]
|
||||
assert long_token not in rows[1]["message"]
|
||||
with pytest.raises(ValueError):
|
||||
repository.record_failure("failure-2", "BAD", "capture-1", "code", "message")
|
||||
|
||||
|
||||
def test_complete_blob_after_capture_insert_failure_is_purgeable_orphan(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"payload-a"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
payload_b = b"payload-b"
|
||||
digest_b = hashlib.sha256(payload_b).hexdigest()
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
repository.receive(
|
||||
"capture-1", "receipt-2", [payload_b], media_type="image/png", max_bytes=100
|
||||
)
|
||||
assert repository.blobs.verify(digest_b) == len(payload_b)
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT count(*) FROM blobs WHERE sha256 = ?", (digest_b,)
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest_b,)
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
assert repository.blobs._path(digest_b).relative_to(
|
||||
repository.blobs.root
|
||||
) in repository.blobs.purge_dry_run(set())
|
||||
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from game_data_capture.config import ServerConfig
|
||||
from game_data_capture.web.app import (
|
||||
CaptureRecord,
|
||||
CaptureSessionStatus,
|
||||
ControlOutcome,
|
||||
MediaDescriptor,
|
||||
MutationOutcome,
|
||||
WebConfig,
|
||||
create_app,
|
||||
)
|
||||
from game_data_capture.web.security import new_bootstrap_token
|
||||
|
||||
_ID = "capture-00000000000000000000000000000002"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptureControl:
|
||||
armed: bool = False
|
||||
|
||||
def status(self) -> CaptureSessionStatus:
|
||||
return CaptureSessionStatus(self.armed, 1, 64)
|
||||
|
||||
def start_session(self) -> ControlOutcome:
|
||||
self.armed = True
|
||||
return "started"
|
||||
|
||||
def stop_session(self) -> ControlOutcome:
|
||||
self.armed = False
|
||||
return "stopped"
|
||||
|
||||
|
||||
class Corpus:
|
||||
def __init__(self) -> None:
|
||||
self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", None)
|
||||
self.rejected = False
|
||||
self.deleted = False
|
||||
|
||||
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
|
||||
return (self.record,)
|
||||
|
||||
def get_capture(self, capture_id: str) -> CaptureRecord | None:
|
||||
return self.record if capture_id == _ID else None
|
||||
|
||||
def open_media(self, capture_id: str) -> MediaDescriptor | None:
|
||||
return MediaDescriptor("image/png", 3, [b"png"]) if capture_id == _ID else None
|
||||
|
||||
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
|
||||
if capture_id != _ID:
|
||||
return "not_found"
|
||||
self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", dict(classification))
|
||||
return "updated"
|
||||
|
||||
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
|
||||
if capture_id != _ID:
|
||||
return "not_found"
|
||||
self.rejected = True
|
||||
return "updated"
|
||||
|
||||
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
|
||||
if capture_id != _ID:
|
||||
return "not_found"
|
||||
self.deleted = True
|
||||
return "updated"
|
||||
|
||||
|
||||
def test_private_capture_labeling_flow_preserves_rejection_before_retention_delete() -> None:
|
||||
token = new_bootstrap_token()
|
||||
control = CaptureControl()
|
||||
corpus = Corpus()
|
||||
client = TestClient(
|
||||
create_app(
|
||||
config=WebConfig(ServerConfig()),
|
||||
capture_control=control,
|
||||
corpus=corpus,
|
||||
bootstrap_token=token,
|
||||
),
|
||||
base_url="http://127.0.0.1:8765",
|
||||
)
|
||||
origin = "http://127.0.0.1:8765"
|
||||
csrf = str(
|
||||
client.post("/api/bootstrap", headers={"Origin": origin}, json={"token": token}).json()[
|
||||
"csrf_token"
|
||||
]
|
||||
)
|
||||
label = {
|
||||
"family": "gear",
|
||||
"view": "detail",
|
||||
"gear_slot": "Mask",
|
||||
"record_types": ["gear", "brand_set"],
|
||||
"confidence": 1.0,
|
||||
"abstention_reason": None,
|
||||
}
|
||||
|
||||
response = client.post(
|
||||
f"/api/captures/{_ID}/label", headers={"Origin": origin, "X-CSRF-Token": csrf}, json=label
|
||||
)
|
||||
assert response.status_code == 200
|
||||
csrf = str(response.json()["csrf_token"])
|
||||
assert client.get(f"/api/captures/{_ID}").json()["capture"]["classification"] == label
|
||||
response = client.post(
|
||||
f"/api/captures/{_ID}/reject",
|
||||
headers={"Origin": origin, "X-CSRF-Token": csrf},
|
||||
json={"confirmation": "REJECT"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
csrf = str(response.json()["csrf_token"])
|
||||
assert corpus.rejected is True
|
||||
response = client.post(
|
||||
f"/api/captures/{_ID}/delete",
|
||||
headers={"Origin": origin, "X-CSRF-Token": csrf},
|
||||
json={"confirmation": "DELETE"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert corpus.deleted is True
|
||||
|
||||
|
||||
def test_review_ui_uses_local_external_assets_and_keeps_guardrails_in_static_source() -> None:
|
||||
web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web"
|
||||
template = (web_root / "templates" / "index.html").read_text(encoding="utf-8")
|
||||
script = (web_root / "static" / "review.js").read_text(encoding="utf-8")
|
||||
stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8")
|
||||
|
||||
assert 'href="/static/review.css"' in template
|
||||
assert 'src="/static/review.js"' in template
|
||||
assert "<style" not in template
|
||||
assert "<script>" not in template
|
||||
assert "http://" not in template + script + stylesheet
|
||||
assert "https://" not in template + script + stylesheet
|
||||
assert "innerHTML" not in script
|
||||
assert "Origin" not in script
|
||||
assert (
|
||||
'history.replaceState(null, "", `${window.location.pathname}${window.location.search}`)'
|
||||
in script
|
||||
)
|
||||
assert '"/api/session/start"' in script
|
||||
assert "JSON.stringify({ confirmation })" in script
|
||||
assert "confidence: 1.0" in script
|
||||
assert 'credentials: "same-origin"' in script
|
||||
assert "localStorage" not in script
|
||||
assert "sessionStorage" not in script
|
||||
for expected in (
|
||||
"weapon",
|
||||
"gear",
|
||||
"skill_chip",
|
||||
"os_protocol",
|
||||
"specialization",
|
||||
"unknown",
|
||||
"Mask",
|
||||
"Gloves",
|
||||
"Backpack",
|
||||
"Body Armor",
|
||||
"Holster",
|
||||
"Knee Pads",
|
||||
"prefers-reduced-motion",
|
||||
):
|
||||
assert expected in script + stylesheet
|
||||
|
||||
|
||||
def test_review_page_and_local_static_assets_serve_with_security_headers() -> None:
|
||||
client = TestClient(
|
||||
create_app(
|
||||
config=WebConfig(ServerConfig()),
|
||||
capture_control=CaptureControl(),
|
||||
corpus=Corpus(),
|
||||
bootstrap_token=new_bootstrap_token(),
|
||||
),
|
||||
base_url="http://127.0.0.1:8765",
|
||||
)
|
||||
|
||||
page = client.get("/")
|
||||
stylesheet = client.get("/static/review.css")
|
||||
script = client.get("/static/review.js")
|
||||
|
||||
for response in (page, stylesheet, script):
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-security-policy"].startswith("default-src 'self'")
|
||||
assert response.headers["referrer-policy"] == "no-referrer"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
assert response.headers["x-frame-options"] == "DENY"
|
||||
assert "http://" not in response.text
|
||||
assert "https://" not in response.text
|
||||
assert 'href="/static/review.css"' in page.text
|
||||
assert 'src="/static/review.js"' in page.text
|
||||
assert stylesheet.headers["content-type"].startswith("text/css")
|
||||
assert "csrfPattern" in script.text
|
||||
assert "setCheckedMembers" in script.text
|
||||
assert "announce: false" in script.text
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.ingest.clipboard import ClipboardIntake, WatchConfig
|
||||
|
||||
|
||||
def test_shutdown_reaps_real_child_process_group(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
child: subprocess.Popen[bytes] | None = None
|
||||
real_popen = subprocess.Popen
|
||||
|
||||
def fake_popen(args: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
|
||||
nonlocal child
|
||||
assert args[0] == "wl-paste" and kwargs["shell"] is False
|
||||
child = real_popen(
|
||||
[sys.executable, "-c", "import time; time.sleep(30)"],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
return child
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", fake_popen)
|
||||
intake = ClipboardIntake(
|
||||
WatchConfig((tmp_path / "spool").resolve(), 100, 1, "v1", 1, 0.01, 0.2)
|
||||
)
|
||||
intake.arm()
|
||||
assert child is not None and child.poll() is None
|
||||
intake.shutdown()
|
||||
for _ in range(20):
|
||||
if child.poll() is not None:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert child.poll() is not None
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_review_interface_marks_keyboard_modal_and_responsive_state_hooks() -> None:
|
||||
web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web"
|
||||
template = (web_root / "templates" / "index.html").read_text(encoding="utf-8")
|
||||
script = (web_root / "static" / "review.js").read_text(encoding="utf-8")
|
||||
stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8")
|
||||
|
||||
assert 'aria-live="polite"' in template
|
||||
assert '<dialog class="confirmation-dialog"' in template
|
||||
assert 'value="cancel"' in template
|
||||
assert 'type="radio"' not in template
|
||||
assert "input.type = type" in script
|
||||
assert 'input.type = "button"' not in script
|
||||
assert 'input.type = "radio"' not in script
|
||||
assert 'input.type = "checkbox"' not in script
|
||||
assert "input.type = type" in script
|
||||
assert "els.dialog.showModal()" in script
|
||||
assert 'els.dialog.returnValue = "cancel"' in script
|
||||
assert 'els.dialog.dataset.action = ""' in script
|
||||
assert "const focusTarget = returnFocus;" in script
|
||||
assert "if (focusTarget) focusTarget.focus();" in script
|
||||
assert "fallbackFocus?.focus();" in script
|
||||
assert 'els.media.addEventListener("error"' in script
|
||||
assert ":focus-visible" in stylesheet
|
||||
assert "min-height: 44px" in stylesheet
|
||||
assert "@media (max-width: 52rem)" in stylesheet
|
||||
assert "@media (prefers-reduced-motion: reduce)" in stylesheet
|
||||
assert "@media (forced-colors: active)" in stylesheet
|
||||
|
||||
|
||||
def test_review_script_guards_session_taxonomy_pending_and_selection_state() -> None:
|
||||
script = (
|
||||
Path(__file__).parents[2] / "src" / "game_data_capture" / "web" / "static" / "review.js"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert 'headers.set("Origin"' not in script
|
||||
assert "Origin:" not in script
|
||||
assert 'const bootstrap = await request("/api/bootstrap"' in script
|
||||
assert 'const session = await request("/api/session")' in script
|
||||
assert script.index('"/api/bootstrap"') < script.index('"/api/session"')
|
||||
assert "if (!validSession(session))" in script
|
||||
assert "family, false, () =>" in script
|
||||
assert "view, false, renderDependentFields" in script
|
||||
assert "if (!classification) { renderViews(); renderDependentFields(); return; }" in script
|
||||
assert "function routeForAction(action)" in script
|
||||
assert '(action === "reject" || action === "delete") && activeCapture' in script
|
||||
assert "const routes =" not in script
|
||||
assert 'document.querySelectorAll("#label-form input")' in script
|
||||
assert "input.disabled = pending || locked || !activeCapture" in script
|
||||
assert 'document.querySelectorAll("#capture-list button")' in script
|
||||
assert "button.disabled = pending || locked;" in script
|
||||
assert "armed === true" in script
|
||||
assert "armed !== true" in script
|
||||
assert "function acceptMutationCsrf(payload)" in script
|
||||
assert "if (!payload || !acceptMutationCsrf(payload))" in script
|
||||
assert 'retained ? "RETAINED"' in script
|
||||
assert "function setCheckedMembers(name, values)" in script
|
||||
assert "input.checked = selectedValues.has(input.value);" in script
|
||||
assert 'setCheckedMembers("record_type", classification.record_types || []);' in script
|
||||
assert 'forEach((record) => setChecked("record_type", record))' not in script
|
||||
assert (
|
||||
"async function refreshCaptures({ announce = true, fallbackFocus = null } = {})" in script
|
||||
)
|
||||
assert "async function selectCapture(captureId, { announce = true } = {})" in script
|
||||
assert 'if (announce) notice("success", "QUEUE REFRESHED"' in script
|
||||
assert 'if (announce) notice("success", "CAPTURE READY"' in script
|
||||
assert "refreshCaptures({ announce: false })" in script
|
||||
assert "selectCapture(captureId, { announce: false })" in script
|
||||
assert script.index("selectCapture(captureId, { announce: false })") < script.index(
|
||||
'notice("success", "LABEL SAVED"'
|
||||
)
|
||||
assert 'if (!activeCapture) els.reviewState.textContent = "READY";' in script
|
||||
assert '"Capture is ARMED. Stop Session requires explicit confirmation."' in script
|
||||
assert '"Capture is DISARMED. Start Session requires explicit confirmation."' in script
|
||||
assert 'action === "start" ? els.stop' in script
|
||||
assert 'action === "stop" ? els.start' in script
|
||||
assert 'action === "delete" ? els.refresh' in script
|
||||
assert "function focusIfAvailable(control)" in script
|
||||
assert "focusIfAvailable(successFocus);" in script
|
||||
assert "async function confirmed(action, fallbackFocus)" in script
|
||||
assert "fallbackFocus?.focus();" in script
|
||||
assert "function clearActiveDetail()" in script
|
||||
assert 'if (action === "delete") clearActiveDetail();' in script
|
||||
assert "captures.some((capture) => capture.capture_id === captureId)" in script
|
||||
assert "refreshCaptures({ fallbackFocus: els.refresh })" in script
|
||||
assert "focusIfAvailable(els.save);" in script
|
||||
@@ -0,0 +1,496 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import subprocess
|
||||
from collections import deque
|
||||
from collections.abc import Iterable, Mapping
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
import uvicorn
|
||||
|
||||
import game_data_capture.cli as cli
|
||||
import game_data_capture.runtime as runtime_module
|
||||
from game_data_capture.config import AppConfig, LifecycleConfig, ServerConfig
|
||||
from game_data_capture.ingest.clipboard import (
|
||||
ClipboardIntake,
|
||||
FocusedWindowMetadata,
|
||||
IntakeReceipt,
|
||||
WatchConfig,
|
||||
)
|
||||
from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, receipt_name, spool_png
|
||||
from game_data_capture.paths import RuntimePaths
|
||||
from game_data_capture.runtime import CaptureRuntime, InstanceLock, RuntimeFailure, _CaptureControl
|
||||
from game_data_capture.storage import CaptureRepository, ContentAddressedBlobStore, StorageDatabase
|
||||
|
||||
_PNG = PNG_SIGNATURE + b"watch-png"
|
||||
|
||||
|
||||
class FakeIntake:
|
||||
def __init__(self, cancelled: Event, payloads: list[dict[str, object]]) -> None:
|
||||
self.cancelled = cancelled
|
||||
self.payloads = deque(payloads)
|
||||
self.receipts: deque[IntakeReceipt] = deque()
|
||||
self.armed = self.reconciled = self.shutdown_called = False
|
||||
self.opened = 0
|
||||
self.acknowledged: list[str] = []
|
||||
self.seen_tokens: set[str] = set()
|
||||
self._ticks = 0
|
||||
|
||||
def reconcile(self) -> list[str]:
|
||||
self.reconciled = True
|
||||
return []
|
||||
|
||||
def arm(self) -> None:
|
||||
self.armed = True
|
||||
|
||||
def tick(self, now: float | None = None) -> None:
|
||||
self._ticks += 1
|
||||
if self._ticks > 1:
|
||||
self.cancelled.set()
|
||||
|
||||
def take_helper_receipt(self) -> dict[str, object] | None:
|
||||
return self.payloads.popleft() if self.payloads else None
|
||||
|
||||
def accept_helper_receipt(
|
||||
self,
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
occurrence_id: str,
|
||||
processing_version: str,
|
||||
) -> bool:
|
||||
token = str(payload["receipt_token"])
|
||||
if token in self.seen_tokens:
|
||||
return False
|
||||
self.seen_tokens.add(token)
|
||||
self.receipts.append(
|
||||
IntakeReceipt(
|
||||
occurrence_id,
|
||||
token,
|
||||
"0" * 64,
|
||||
len(_PNG),
|
||||
"image/png",
|
||||
Path("/private/spool.png"),
|
||||
"capture-v1",
|
||||
processing_version,
|
||||
FocusedWindowMetadata(None, None, None),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def take(self) -> IntakeReceipt | None:
|
||||
return self.receipts.popleft() if self.receipts else None
|
||||
|
||||
def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]:
|
||||
self.opened += 1
|
||||
return nullcontext([_PNG])
|
||||
|
||||
def acknowledge(self, receipt: IntakeReceipt) -> bool:
|
||||
self.acknowledged.append(receipt.receipt_token)
|
||||
return True
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.shutdown_called = True
|
||||
|
||||
|
||||
def _runtime(tmp_path: Path, intake: FakeIntake) -> CaptureRuntime:
|
||||
root = tmp_path / "xdg"
|
||||
paths = RuntimePaths(
|
||||
config_dir=(root / "config").resolve(),
|
||||
data_dir=(root / "data").resolve(),
|
||||
cache_dir=(root / "cache").resolve(),
|
||||
runtime_dir=(root / "runtime").resolve(),
|
||||
)
|
||||
return CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
paths,
|
||||
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
|
||||
intake_factory=lambda _config: intake,
|
||||
)
|
||||
|
||||
|
||||
def _payload(token: str) -> dict[str, object]:
|
||||
return {
|
||||
"receipt_version": 1,
|
||||
"receipt_token": token,
|
||||
"sha256": "0" * 64,
|
||||
"byte_size": len(_PNG),
|
||||
"media_type": "image/png",
|
||||
"spool_name": "ignored.png",
|
||||
"policy_version": "capture-v1",
|
||||
}
|
||||
|
||||
|
||||
class PassiveClipboardIntake(ClipboardIntake):
|
||||
"""Use real private spool sidecars without launching a Wayland watcher."""
|
||||
|
||||
def __init__(
|
||||
self, config: WatchConfig, cancelled: Event, *, acknowledge_result: bool = True
|
||||
) -> None:
|
||||
super().__init__(config)
|
||||
self.cancelled = cancelled
|
||||
self.acknowledge_result = acknowledge_result
|
||||
self.acknowledged: list[str] = []
|
||||
self.reconcile_calls = 0
|
||||
self._ticks = 0
|
||||
|
||||
def reconcile(self) -> list[str]:
|
||||
self.reconcile_calls += 1
|
||||
return super().reconcile()
|
||||
|
||||
def arm(self) -> None:
|
||||
self.armed, self.paused = True, False
|
||||
|
||||
def tick(self, now: float | None = None) -> None:
|
||||
self._ticks += 1
|
||||
if self._ticks > 1:
|
||||
self.cancelled.set()
|
||||
|
||||
def acknowledge(self, receipt: IntakeReceipt) -> bool:
|
||||
self.acknowledged.append(receipt.receipt_token)
|
||||
return self.acknowledge_result and super().acknowledge(receipt)
|
||||
|
||||
|
||||
class StopTimeoutReceiptIntake(PassiveClipboardIntake):
|
||||
def __init__(self, config: WatchConfig, cancelled: Event) -> None:
|
||||
super().__init__(config, cancelled)
|
||||
self.tick_entered = Event()
|
||||
self.release_tick = Event()
|
||||
|
||||
def tick(self, now: float | None = None) -> None:
|
||||
self.tick_entered.set()
|
||||
self.release_tick.wait()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.release_tick.set()
|
||||
super().shutdown()
|
||||
|
||||
|
||||
def _durable_runtime(
|
||||
tmp_path: Path, cancelled: Event, *, acknowledge_result: bool = True
|
||||
) -> tuple[CaptureRuntime, list[PassiveClipboardIntake]]:
|
||||
created: list[PassiveClipboardIntake] = []
|
||||
root = tmp_path / "durable-xdg"
|
||||
paths = RuntimePaths(
|
||||
config_dir=(root / "config").resolve(),
|
||||
data_dir=(root / "data").resolve(),
|
||||
cache_dir=(root / "cache").resolve(),
|
||||
runtime_dir=(root / "runtime").resolve(),
|
||||
)
|
||||
|
||||
def factory(config: WatchConfig) -> PassiveClipboardIntake:
|
||||
intake = PassiveClipboardIntake(config, cancelled, acknowledge_result=acknowledge_result)
|
||||
created.append(intake)
|
||||
return intake
|
||||
|
||||
return (
|
||||
CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)),
|
||||
paths,
|
||||
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
|
||||
intake_factory=factory,
|
||||
),
|
||||
created,
|
||||
)
|
||||
|
||||
|
||||
def _durable_receipt(runtime: CaptureRuntime, token: str) -> None:
|
||||
spool_png(
|
||||
io.BytesIO(_PNG),
|
||||
spool=runtime.paths.cache_dir / "spool",
|
||||
max_bytes=100,
|
||||
receipt_token=token,
|
||||
policy_version="capture-v1",
|
||||
)
|
||||
|
||||
|
||||
def test_watch_streams_verified_receipts_and_replays_exact_tokens_idempotently(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
cancelled = Event()
|
||||
intake = FakeIntake(cancelled, [_payload("watch-receipt-1"), _payload("watch-receipt-1")])
|
||||
runtime = _runtime(tmp_path, intake)
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir)
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
assert intake.armed and intake.reconciled and intake.shutdown_called and intake.opened == 1
|
||||
assert intake.acknowledged == ["watch-receipt-1"]
|
||||
|
||||
|
||||
def test_watch_ignores_an_absent_helper_receipt_without_error_noise(tmp_path: Path) -> None:
|
||||
cancelled = Event()
|
||||
intake = FakeIntake(cancelled, [])
|
||||
runtime = _runtime(tmp_path, intake)
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir)
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert intake.shutdown_called and intake.opened == 0
|
||||
|
||||
|
||||
def test_startup_reconciles_partial_blob_storage_before_arming(tmp_path: Path) -> None:
|
||||
cancelled = Event()
|
||||
intake = FakeIntake(cancelled, [])
|
||||
runtime = _runtime(tmp_path, intake)
|
||||
blobs = ContentAddressedBlobStore(runtime.paths.data_dir)
|
||||
blobs.temporary.mkdir(parents=True)
|
||||
partial = blobs.temporary / ".leftover.partial"
|
||||
partial.write_bytes(b"partial")
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
assert not partial.exists()
|
||||
assert intake.reconciled and intake.shutdown_called
|
||||
|
||||
|
||||
def test_durable_receipt_is_acknowledged_once_after_repository_confirmation(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
cancelled = Event()
|
||||
runtime, created = _durable_runtime(tmp_path, cancelled)
|
||||
_durable_receipt(runtime, "confirmed")
|
||||
events: list[str] = []
|
||||
original_receive = CaptureRepository.receive
|
||||
original_acknowledge = PassiveClipboardIntake.acknowledge
|
||||
|
||||
def tracked_receive(
|
||||
self: CaptureRepository,
|
||||
capture_id: str,
|
||||
receipt_token: str,
|
||||
chunks: Iterable[bytes],
|
||||
*,
|
||||
media_type: str,
|
||||
max_bytes: int,
|
||||
) -> str:
|
||||
events.append("receive")
|
||||
return original_receive(
|
||||
self,
|
||||
capture_id,
|
||||
receipt_token,
|
||||
chunks,
|
||||
media_type=media_type,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(CaptureRepository, "receive", tracked_receive)
|
||||
|
||||
def tracked_acknowledge(self: PassiveClipboardIntake, receipt: IntakeReceipt) -> bool:
|
||||
events.append("acknowledge")
|
||||
return original_acknowledge(self, receipt)
|
||||
|
||||
monkeypatch.setattr(PassiveClipboardIntake, "acknowledge", tracked_acknowledge)
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
|
||||
assert events == ["receive", "acknowledge"]
|
||||
assert created[0].acknowledged == ["confirmed"]
|
||||
assert created[0].reconcile_calls == 1
|
||||
assert not (runtime.paths.cache_dir / "spool" / receipt_name("confirmed")).exists()
|
||||
|
||||
|
||||
def test_storage_failure_leaves_durable_receipt_unacknowledged(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
cancelled = Event()
|
||||
runtime, created = _durable_runtime(tmp_path, cancelled)
|
||||
_durable_receipt(runtime, "storage-failure")
|
||||
|
||||
def failed_receive(
|
||||
self: CaptureRepository,
|
||||
capture_id: str,
|
||||
receipt_token: str,
|
||||
chunks: Iterable[bytes],
|
||||
*,
|
||||
media_type: str,
|
||||
max_bytes: int,
|
||||
) -> str:
|
||||
raise OSError("storage unavailable")
|
||||
|
||||
monkeypatch.setattr(CaptureRepository, "receive", failed_receive)
|
||||
with pytest.raises(RuntimeFailure, match="capture watch failed"):
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
|
||||
assert created[0].acknowledged == []
|
||||
assert (runtime.paths.cache_dir / "spool" / receipt_name("storage-failure")).exists()
|
||||
|
||||
|
||||
def test_cancellation_after_storage_confirmation_does_not_acknowledge(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
cancelled = Event()
|
||||
runtime, created = _durable_runtime(tmp_path, cancelled)
|
||||
_durable_receipt(runtime, "cancelled-after-storage")
|
||||
original_receive = CaptureRepository.receive
|
||||
|
||||
def cancel_after_receive(
|
||||
self: CaptureRepository,
|
||||
capture_id: str,
|
||||
receipt_token: str,
|
||||
chunks: Iterable[bytes],
|
||||
*,
|
||||
media_type: str,
|
||||
max_bytes: int,
|
||||
) -> str:
|
||||
result = original_receive(
|
||||
self,
|
||||
capture_id,
|
||||
receipt_token,
|
||||
chunks,
|
||||
media_type=media_type,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
cancelled.set()
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(CaptureRepository, "receive", cancel_after_receive)
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
|
||||
assert created[0].acknowledged == []
|
||||
assert (runtime.paths.cache_dir / "spool" / receipt_name("cancelled-after-storage")).exists()
|
||||
|
||||
|
||||
def test_acknowledgement_failure_replays_idempotently_and_releases_lock(tmp_path: Path) -> None:
|
||||
cancelled = Event()
|
||||
runtime, failed = _durable_runtime(tmp_path, cancelled, acknowledge_result=False)
|
||||
_durable_receipt(runtime, "replay-token")
|
||||
with pytest.raises(RuntimeFailure, match="acknowledgement failed"):
|
||||
runtime.watch(cancelled=cancelled, poll_seconds=0.001)
|
||||
assert failed[0].acknowledged == ["replay-token"]
|
||||
sidecar = runtime.paths.cache_dir / "spool" / receipt_name("replay-token")
|
||||
assert sidecar.exists()
|
||||
reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0)
|
||||
reusable.release()
|
||||
|
||||
replay_cancelled = Event()
|
||||
replay_runtime, replayed = _durable_runtime(tmp_path, replay_cancelled)
|
||||
replay_runtime.watch(cancelled=replay_cancelled, poll_seconds=0.001)
|
||||
database = StorageDatabase(replay_runtime.paths.data_dir, replay_runtime.migrations_dir)
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert replayed[0].acknowledged == ["replay-token"]
|
||||
assert not sidecar.exists()
|
||||
|
||||
|
||||
def test_service_stop_timeout_preserves_unacknowledged_durable_sidecar(tmp_path: Path) -> None:
|
||||
cancelled = Event()
|
||||
created: list[StopTimeoutReceiptIntake] = []
|
||||
root = tmp_path / "stop-timeout-xdg"
|
||||
paths = RuntimePaths(
|
||||
config_dir=(root / "config").resolve(),
|
||||
data_dir=(root / "data").resolve(),
|
||||
cache_dir=(root / "cache").resolve(),
|
||||
runtime_dir=(root / "runtime").resolve(),
|
||||
)
|
||||
|
||||
def factory(config: WatchConfig) -> StopTimeoutReceiptIntake:
|
||||
intake = StopTimeoutReceiptIntake(config, cancelled)
|
||||
created.append(intake)
|
||||
return intake
|
||||
|
||||
runtime = CaptureRuntime(
|
||||
AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0, shutdown_timeout_seconds=1)),
|
||||
paths,
|
||||
migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(),
|
||||
intake_factory=factory,
|
||||
)
|
||||
_durable_receipt(runtime, "timeout-receipt")
|
||||
resources = runtime._start()
|
||||
control = _CaptureControl(runtime, resources.repository)
|
||||
try:
|
||||
assert control.start_session() == "started"
|
||||
assert created[0].tick_entered.wait(0.2)
|
||||
assert control.stop_session() == "stopped"
|
||||
assert created[0].acknowledged == []
|
||||
assert (runtime.paths.cache_dir / "spool" / receipt_name("timeout-receipt")).exists()
|
||||
finally:
|
||||
control.shutdown()
|
||||
resources.close()
|
||||
|
||||
|
||||
def test_cli_bare_watch_is_a_safe_refusal_and_modes_never_start_live_processes(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
root = tmp_path / "fresh-xdg"
|
||||
for key, name in (
|
||||
("HOME", "home"),
|
||||
("XDG_CONFIG_HOME", "config"),
|
||||
("XDG_DATA_HOME", "data"),
|
||||
("XDG_CACHE_HOME", "cache"),
|
||||
("XDG_RUNTIME_DIR", "runtime"),
|
||||
):
|
||||
monkeypatch.setenv(key, str(root / name))
|
||||
|
||||
def forbidden(*_arguments: object, **_kwargs: object) -> None:
|
||||
raise AssertionError("test started a process")
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", forbidden)
|
||||
assert cli.main(["watch"]) == cli.EXIT_UNIMPLEMENTED
|
||||
assert "requires gdc-runtime-integration" in capsys.readouterr().err
|
||||
assert cli.main(["watch", "--dry-run"]) == 0
|
||||
assert capsys.readouterr().out == "watch dry-run: capture runtime plan is valid\n"
|
||||
assert not root.exists()
|
||||
|
||||
|
||||
def test_cli_arm_reaches_only_a_mocked_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, _config: object, _paths: object) -> None:
|
||||
calls.append("created")
|
||||
|
||||
def watch(self) -> None:
|
||||
calls.append("watch")
|
||||
|
||||
monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime)
|
||||
assert cli.main(["watch", "--arm"]) == 0
|
||||
assert calls == ["created", "watch"]
|
||||
|
||||
|
||||
def test_cli_serve_validates_before_constructing_resources_and_passes_exact_authority(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, config: AppConfig, _paths: object) -> None:
|
||||
calls.append(("created", config.server))
|
||||
|
||||
def serve(self) -> None:
|
||||
calls.append(("serve", None))
|
||||
|
||||
monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime)
|
||||
assert cli.main(["serve", "--host", "127.0.0.1", "--port", "9877"]) == 0
|
||||
assert calls == [("created", ServerConfig("127.0.0.1", 9877)), ("serve", None)]
|
||||
|
||||
calls.clear()
|
||||
assert cli.main(["serve", "--host", "0.0.0.0"]) == cli.EXIT_CONFIG_ERROR
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_cli_serve_converts_uvicorn_startup_exit_without_leaking_its_token(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
root = tmp_path / "serve-xdg"
|
||||
for key, name in (
|
||||
("HOME", "home"),
|
||||
("XDG_CONFIG_HOME", "config"),
|
||||
("XDG_DATA_HOME", "data"),
|
||||
("XDG_CACHE_HOME", "cache"),
|
||||
("XDG_RUNTIME_DIR", "runtime"),
|
||||
):
|
||||
monkeypatch.setenv(key, str(root / name))
|
||||
token = "Z" * 43
|
||||
monkeypatch.setattr(runtime_module, "new_bootstrap_token", lambda: token)
|
||||
|
||||
def failed_server(*_: object, **__: object) -> None:
|
||||
raise SystemExit(2)
|
||||
|
||||
monkeypatch.setattr(uvicorn, "run", failed_server)
|
||||
assert cli.main(["serve"]) == cli.EXIT_RUNTIME_ERROR
|
||||
output = capsys.readouterr()
|
||||
assert output.err == "capture runtime failed\n"
|
||||
assert token not in output.err
|
||||
assert "SystemExit" not in output.err
|
||||
reusable = InstanceLock.acquire(root / "runtime" / "shd-capture" / "instance.lock", 0)
|
||||
reusable.release()
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from game_data_capture.config import ServerConfig
|
||||
from game_data_capture.web.app import (
|
||||
CaptureRecord,
|
||||
CaptureSessionStatus,
|
||||
ControlOutcome,
|
||||
MediaDescriptor,
|
||||
MutationOutcome,
|
||||
WebConfig,
|
||||
create_app,
|
||||
)
|
||||
from game_data_capture.web.security import new_bootstrap_token
|
||||
|
||||
|
||||
class Control:
|
||||
def status(self) -> CaptureSessionStatus:
|
||||
return CaptureSessionStatus(False, 0, 1)
|
||||
|
||||
def start_session(self) -> ControlOutcome:
|
||||
return "started"
|
||||
|
||||
def stop_session(self) -> ControlOutcome:
|
||||
return "stopped"
|
||||
|
||||
|
||||
class EmptyCorpus:
|
||||
def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]:
|
||||
return ()
|
||||
|
||||
def get_capture(self, capture_id: str) -> CaptureRecord | None:
|
||||
return None
|
||||
|
||||
def open_media(self, capture_id: str) -> MediaDescriptor | None:
|
||||
return None
|
||||
|
||||
def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
def delete_with_retention(self, capture_id: str) -> MutationOutcome:
|
||||
return "not_found"
|
||||
|
||||
|
||||
def test_bootstrap_is_one_time_same_origin_and_does_not_enable_permissive_cors() -> None:
|
||||
token = new_bootstrap_token()
|
||||
app = create_app(
|
||||
config=WebConfig(ServerConfig()),
|
||||
capture_control=Control(),
|
||||
corpus=EmptyCorpus(),
|
||||
bootstrap_token=token,
|
||||
)
|
||||
client = TestClient(app, base_url="http://127.0.0.1:8765")
|
||||
|
||||
cross_origin = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://evil.test"}, json={"token": token}
|
||||
)
|
||||
accepted = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
|
||||
)
|
||||
replay = client.post(
|
||||
"/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token}
|
||||
)
|
||||
|
||||
assert cross_origin.status_code == 403
|
||||
assert accepted.status_code == 200
|
||||
assert replay.status_code == 401
|
||||
assert "access-control-allow-origin" not in accepted.headers
|
||||
Reference in New Issue
Block a user