Add game data capture vertical slice
This commit is contained in:
@@ -0,0 +1,458 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.storage import CaptureCorpus, CaptureRepository, ContentAddressedBlobStore
|
||||
from game_data_capture.storage.blobs import BlobCorruptionError, BlobLockError
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import StorageError
|
||||
|
||||
|
||||
def _label(*, view: str = "detail", records: list[str] | None = None) -> dict[str, object]:
|
||||
return {
|
||||
"family": "weapon",
|
||||
"view": view,
|
||||
"gear_slot": None,
|
||||
"record_types": ["weapon"] if records is None else records,
|
||||
"confidence": 1.0,
|
||||
"abstention_reason": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corpus(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository, CaptureCorpus]:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
database = StorageDatabase(
|
||||
root,
|
||||
(Path(__file__).parents[3] / "migrations").resolve(),
|
||||
)
|
||||
database.migrate()
|
||||
blobs = ContentAddressedBlobStore(root)
|
||||
repository = CaptureRepository(database, blobs)
|
||||
return database, repository, CaptureCorpus(database, blobs, max_media_bytes=1024)
|
||||
|
||||
|
||||
def _receive(
|
||||
repository: CaptureRepository, capture_id: str, token: str, payload: bytes = b"png"
|
||||
) -> None:
|
||||
repository.receive(capture_id, token, [payload], media_type="image/png", max_bytes=1024)
|
||||
|
||||
|
||||
def _assert_media_unavailable(port: CaptureCorpus, capture_id: str) -> None:
|
||||
media = port.open_media(capture_id)
|
||||
assert media is not None
|
||||
with pytest.raises(BlobCorruptionError):
|
||||
next(iter(media.chunks))
|
||||
|
||||
|
||||
def _commit_deletion_intent(database: StorageDatabase, capture_id: str) -> None:
|
||||
with database.connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT c.blob_sha256, c.state, c.received_at, c.receipt_token, c.failure_reason, "
|
||||
"c.revision, b.byte_size FROM captures AS c JOIN blobs AS b "
|
||||
"ON b.sha256 = c.blob_sha256 WHERE c.capture_id = ?",
|
||||
(capture_id,),
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute(
|
||||
"INSERT INTO capture_deletion_intents(blob_sha256, capture_id, byte_size, "
|
||||
"capture_state, received_at, receipt_token, failure_reason, revision, created_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
row["blob_sha256"],
|
||||
capture_id,
|
||||
row["byte_size"],
|
||||
row["state"],
|
||||
row["received_at"],
|
||||
row["receipt_token"],
|
||||
row["failure_reason"],
|
||||
row["revision"],
|
||||
"time",
|
||||
),
|
||||
)
|
||||
connection.execute("DELETE FROM captures WHERE capture_id = ?", (capture_id,))
|
||||
connection.execute("COMMIT")
|
||||
|
||||
|
||||
def test_safe_bounded_listing_detail_and_verified_media(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
records = port.list_captures(limit=1)
|
||||
assert len(records) == 1 and records[0].capture_id == "capture-0002"
|
||||
assert port.get_capture("../capture-0001") is None
|
||||
assert port.open_media("../capture-0001") is None
|
||||
media = port.open_media("capture-0001")
|
||||
assert media is not None
|
||||
assert media.media_type == "image/png" and media.byte_size == 3
|
||||
chunks = media.chunks
|
||||
assert next(iter(chunks)) == b"png"
|
||||
close = getattr(chunks, "close", None)
|
||||
assert callable(close)
|
||||
close()
|
||||
for invalid in (0, 257, True):
|
||||
with pytest.raises(ValueError):
|
||||
port.list_captures(limit=invalid)
|
||||
|
||||
|
||||
def test_labels_preserve_prior_values_across_restart_and_reject_is_idempotent(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
first = _label()
|
||||
second = _label(view="blueprint", records=["weapon_talent"])
|
||||
with pytest.raises(ValueError):
|
||||
port.label("capture-0001", {"family": "weapon"})
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", second) == "updated"
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
record = restarted.get_capture("capture-0001")
|
||||
assert record is not None and record.classification == second
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
|
||||
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert restarted.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert restarted.label("capture-0001", first) == "conflict"
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_rejections").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_concurrent_label_writes_serialize_without_losing_prior_value(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
labels = (_label(), _label(view="blueprint", records=["weapon_talent"]))
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
outcomes = list(executor.map(lambda value: port.label("capture-0001", value), labels))
|
||||
assert outcomes == ["updated", "updated"]
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_canonical_equivalent_labels_do_not_append_history(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
first = _label(records=["weapon", "weapon_talent"])
|
||||
equivalent = _label(records=["weapon_talent", "weapon"])
|
||||
equivalent["confidence"] = 1
|
||||
assert port.label("capture-0001", first) == "updated"
|
||||
assert port.label("capture-0001", equivalent) == "updated"
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_rejected_or_evidence_bearing_captures_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.reject_preserving_evidence("capture-0001") == "updated"
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
repository.append_ocr_run("run-0002", "capture-0002", "ocr", "1", {})
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
assert port.delete_with_retention("bad") == "not_found"
|
||||
|
||||
|
||||
def test_label_and_job_evidence_retain_capture(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.label("capture-0001", _label()) == "updated"
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO jobs(job_id, capture_id, kind, state, max_attempts) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
("job-0002", "capture-0002", "ocr", "queued", 1),
|
||||
)
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
|
||||
|
||||
def test_receive_translates_put_lock_failure_without_capture(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, _ = corpus
|
||||
|
||||
def locked(_: object, *, max_bytes: int) -> tuple[str, int]:
|
||||
raise BlobLockError("locked")
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "put", locked)
|
||||
with pytest.raises(StorageError, match="unavailable"):
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("state", "failure_reason"),
|
||||
(("failed", None), ("cancelled", None), ("stored", "capture failure"), ("unknown", None)),
|
||||
)
|
||||
def test_non_disposable_capture_rows_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
state: str,
|
||||
failure_reason: str | None,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"UPDATE captures SET state = ?, failure_reason = ? WHERE capture_id = ?",
|
||||
(state, failure_reason, "capture-0001"),
|
||||
)
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
|
||||
|
||||
def test_capture_lineage_and_failure_evidence_are_retained(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
repository.append_lineage("lineage-0001", "capture-0001", "parent-0001", "derived")
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
_receive(repository, "capture-0002", "receipt-0002")
|
||||
repository.record_failure("failure-0002", "capture", "capture-0002", "fault", "bounded")
|
||||
assert port.delete_with_retention("capture-0002") == "retained"
|
||||
_receive(repository, "capture-0003", "receipt-0003")
|
||||
repository.append_lineage("lineage-0003", "child-0003", "capture-0003", "derived")
|
||||
assert port.delete_with_retention("capture-0003") == "retained"
|
||||
|
||||
|
||||
def test_shared_blob_deletion_keeps_remaining_capture_media(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001", b"shared")
|
||||
_receive(repository, "capture-0002", "receipt-0002", b"shared")
|
||||
assert port.delete_with_retention("capture-0001") == "updated"
|
||||
assert port.get_capture("capture-0001") is None
|
||||
media = port.open_media("capture-0002")
|
||||
assert media is not None and b"".join(media.chunks) == b"shared"
|
||||
with repository.database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_unshared_deletion_removes_media_metadata_and_intent(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
assert port.delete_with_retention("capture-0001") == "updated"
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
|
||||
|
||||
def test_committed_intent_before_unlink_blocks_replay_then_finalizes(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
_commit_deletion_intent(database, "capture-0001")
|
||||
assert path.exists()
|
||||
with pytest.raises(StorageError, match="deletion"):
|
||||
_receive(repository, "capture-0002", "receipt-0001")
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
assert restarted.reconcile_deletions() == 1
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
|
||||
|
||||
def test_interleaved_reinstall_waits_for_deletion_finalization(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001", b"shared")
|
||||
unlinked = threading.Event()
|
||||
release = threading.Event()
|
||||
original = repository.blobs.delete_verified
|
||||
outcomes: list[object] = []
|
||||
|
||||
def pause(digest: str, *, expected_size: int) -> bool:
|
||||
result = original(digest, expected_size=expected_size)
|
||||
unlinked.set()
|
||||
assert release.wait(0.5)
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "delete_verified", pause)
|
||||
deletion = threading.Thread(
|
||||
target=lambda: outcomes.append(port.delete_with_retention("capture-0001"))
|
||||
)
|
||||
deletion.start()
|
||||
assert unlinked.wait(0.5)
|
||||
receipt = threading.Thread(
|
||||
target=lambda: outcomes.append(
|
||||
repository.receive(
|
||||
"capture-0002", "receipt-0002", [b"shared"], media_type="image/png", max_bytes=1024
|
||||
)
|
||||
)
|
||||
)
|
||||
receipt.start()
|
||||
release.set()
|
||||
deletion.join(1)
|
||||
receipt.join(1)
|
||||
assert not deletion.is_alive() and not receipt.is_alive()
|
||||
assert set(outcomes) == {"updated", "capture-0002"}
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
media = port.open_media("capture-0002")
|
||||
assert media is not None and b"".join(media.chunks) == b"shared"
|
||||
|
||||
|
||||
def test_unlink_then_sqlite_finalization_failure_recovers(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
row = database.connect().execute("SELECT sha256, byte_size FROM blobs").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["sha256"]))
|
||||
_commit_deletion_intent(database, "capture-0001")
|
||||
repository.blobs.delete_verified(str(row["sha256"]), expected_size=int(row["byte_size"]))
|
||||
assert not path.exists()
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"CREATE TRIGGER fail_finalization BEFORE DELETE ON capture_deletion_intents "
|
||||
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
|
||||
)
|
||||
with pytest.raises(sqlite3.IntegrityError, match="fault"):
|
||||
port.reconcile_deletions()
|
||||
assert port.get_capture("capture-0001") is None
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 1
|
||||
)
|
||||
connection.execute("DROP TRIGGER fail_finalization")
|
||||
restarted = CaptureCorpus(
|
||||
database, ContentAddressedBlobStore(database.root), max_media_bytes=1024
|
||||
)
|
||||
assert restarted.reconcile_deletions() == 1
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_corrupt_missing_and_symlink_media_fail_closed(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], tmp_path: Path
|
||||
) -> None:
|
||||
_, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with repository.database.connect() as connection:
|
||||
row = connection.execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None
|
||||
path = repository.blobs._path(str(row["blob_sha256"]))
|
||||
path.write_bytes(b"x" * 2048)
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
path.unlink()
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
path.symlink_to(tmp_path / "outside")
|
||||
_assert_media_unavailable(port, "capture-0001")
|
||||
|
||||
|
||||
def test_label_transaction_rolls_back_on_evidence_failure(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
assert port.label("capture-0001", _label()) == "updated"
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"CREATE TRIGGER fail_label_evidence BEFORE INSERT ON capture_label_evidence "
|
||||
"BEGIN SELECT RAISE(ABORT, 'fault'); END"
|
||||
)
|
||||
with pytest.raises(sqlite3.IntegrityError, match="fault"):
|
||||
port.label(
|
||||
"capture-0001",
|
||||
_label(view="blueprint", records=["weapon_talent"]),
|
||||
)
|
||||
assert port.get_capture("capture-0001") is not None
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0
|
||||
stored = connection.execute("SELECT classification_json FROM capture_labels").fetchone()
|
||||
assert stored is not None and '"view":"detail"' in stored["classification_json"]
|
||||
|
||||
|
||||
def test_deletion_intent_recovers_after_unlink_failure(
|
||||
corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database, repository, port = corpus
|
||||
_receive(repository, "capture-0001", "receipt-0001")
|
||||
with database.connect() as connection:
|
||||
original = tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
def fail_unlink(_: str, *, expected_size: int) -> bool:
|
||||
raise BlobCorruptionError("injected fault")
|
||||
|
||||
monkeypatch.setattr(repository.blobs, "delete_verified", fail_unlink)
|
||||
assert port.delete_with_retention("capture-0001") == "retained"
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert (
|
||||
tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
)
|
||||
== original
|
||||
)
|
||||
assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1
|
||||
media = port.open_media("capture-0001")
|
||||
assert media is not None and b"".join(media.chunks) == b"png"
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.storage.database import MigrationError, StorageDatabase
|
||||
|
||||
|
||||
def _migrations(tmp_path: Path) -> Path:
|
||||
source = Path(__file__).parents[3] / "migrations" / "001_initial.sql"
|
||||
destination = tmp_path / "migrations"
|
||||
destination.mkdir()
|
||||
shutil.copy(source, destination / source.name)
|
||||
return destination
|
||||
|
||||
|
||||
def test_connection_pragmas_and_private_modes(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
database = StorageDatabase(root, _migrations(tmp_path).resolve())
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS live_wal(value INTEGER)")
|
||||
assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1
|
||||
assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||||
assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000
|
||||
assert connection.execute("PRAGMA synchronous").fetchone()[0] == 2
|
||||
for path in (
|
||||
database.path.with_name("capture.sqlite3-wal"),
|
||||
database.path.with_name("capture.sqlite3-shm"),
|
||||
):
|
||||
if path.exists():
|
||||
assert path.stat().st_mode & 0o077 == 0
|
||||
connection.execute("COMMIT")
|
||||
assert root.stat().st_mode & 0o777 == 0o700
|
||||
assert database.path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_database_symlink_is_rejected(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
root.mkdir()
|
||||
outside = tmp_path / "outside"
|
||||
outside.write_text("x")
|
||||
(root / "capture.sqlite3").symlink_to(outside)
|
||||
with pytest.raises(MigrationError):
|
||||
StorageDatabase(root, _migrations(tmp_path).resolve()).connect()
|
||||
|
||||
|
||||
def test_migration_history_and_failure_rollback(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
first = migrations / "001_initial.sql"
|
||||
first.write_text(first.read_text() + "\n-- changed\n")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
first.write_text((Path(__file__).parents[3] / "migrations" / "001_initial.sql").read_text())
|
||||
bad = migrations / "002_bad.sql"
|
||||
bad.write_text("CREATE TABLE partial_test(value INTEGER); BAD SQL;")
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE name='partial_test'"
|
||||
).fetchone()
|
||||
is None
|
||||
)
|
||||
bad.write_text("CREATE TABLE partial_test(value INTEGER);")
|
||||
database.migrate()
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
|
||||
|
||||
|
||||
def test_migration_removal_rename_and_duplicate_rejected(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
path = migrations / "001_initial.sql"
|
||||
path.rename(migrations / "001_renamed.sql")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
(migrations / "001_renamed.sql").rename(path)
|
||||
path.unlink()
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
shutil.copy(Path(__file__).parents[3] / "migrations" / "001_initial.sql", path)
|
||||
shutil.copy(path, migrations / "001_duplicate.sql")
|
||||
with pytest.raises(MigrationError):
|
||||
database.migrate()
|
||||
|
||||
|
||||
def test_corpus_migration_preserves_populated_initial_database(tmp_path: Path) -> None:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = _migrations(tmp_path).resolve()
|
||||
database = StorageDatabase(root, migrations)
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO blobs(sha256, byte_size, media_type, created_at) VALUES (?, 1, ?, ?)",
|
||||
("0" * 64, "image/png", "time"),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO captures(capture_id, blob_sha256, state, received_at, receipt_token) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
("capture-0001", "0" * 64, "stored", "time", "receipt-0001"),
|
||||
)
|
||||
source = Path(__file__).parents[3] / "migrations" / "002_capture_corpus.sql"
|
||||
shutil.copy(source, migrations / source.name)
|
||||
database.migrate()
|
||||
with database.connect() as connection:
|
||||
assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1
|
||||
assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 0
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT sha256, byte_size, media_type, created_at FROM blobs"
|
||||
).fetchone()
|
||||
) == ("0" * 64, 1, "image/png", "time")
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT capture_id, blob_sha256, state, received_at, receipt_token, "
|
||||
"failure_reason, revision FROM captures"
|
||||
).fetchone()
|
||||
) == ("capture-0001", "0" * 64, "stored", "time", "receipt-0001", None, 0)
|
||||
assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2
|
||||
migration = connection.execute(
|
||||
"SELECT filename, checksum FROM schema_migrations WHERE version = 2"
|
||||
).fetchone()
|
||||
assert migration is not None
|
||||
assert migration["filename"] == "002_capture_corpus.sql"
|
||||
assert isinstance(migration["checksum"], str) and len(migration["checksum"]) == 64
|
||||
names = {row["name"] for row in connection.execute("SELECT name FROM sqlite_master")}
|
||||
assert {
|
||||
"captures_blob_sha256",
|
||||
"jobs_capture_id",
|
||||
"ocr_runs_capture_id",
|
||||
"runtime_failures_capture_subject",
|
||||
"lineage_child_id",
|
||||
"lineage_parent_id",
|
||||
}.issubset(names)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.pipeline.jobs import JobLeases
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import StorageError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database(tmp_path: Path) -> StorageDatabase:
|
||||
database = StorageDatabase(
|
||||
(tmp_path / "utility").resolve(), (Path(__file__).parents[3] / "migrations").resolve()
|
||||
)
|
||||
database.migrate()
|
||||
return database
|
||||
|
||||
|
||||
def _expire(database: StorageDatabase, job_id: str) -> None:
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"UPDATE jobs SET lease_expires_at = ? WHERE job_id = ?",
|
||||
((datetime.now(UTC) - timedelta(seconds=1)).isoformat(), job_id),
|
||||
)
|
||||
|
||||
|
||||
def test_job_validation_and_exclusive_ownership(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
for job_id, kind, attempts in (
|
||||
("!", "ocr", 1),
|
||||
("job", "bad", 1),
|
||||
("job", "ocr", True),
|
||||
("job", "ocr", 0),
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.create(job_id, kind, max_attempts=attempts)
|
||||
jobs.create("job-1", "ocr")
|
||||
for duration in (0, 301, True):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.acquire("owner-1", duration)
|
||||
with pytest.raises(ValueError):
|
||||
jobs.acquire("!", 30)
|
||||
assert jobs.acquire("owner-1", 30) == "job-1"
|
||||
assert jobs.acquire("owner-2", 30) is None
|
||||
with pytest.raises(StorageError):
|
||||
jobs.renew("job-1", "owner-2", 30)
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("job-1", "owner-2", "succeeded")
|
||||
for duration in (0, 301, True):
|
||||
with pytest.raises(ValueError):
|
||||
jobs.renew("job-1", "owner-1", duration)
|
||||
|
||||
|
||||
def test_expiry_recovery_exhaustion_and_terminal_jobs(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("retry", "ocr", max_attempts=2)
|
||||
assert jobs.acquire("owner-1", 30) == "retry"
|
||||
_expire(database, "retry")
|
||||
with pytest.raises(StorageError):
|
||||
jobs.renew("retry", "owner-1", 30)
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("retry", "owner-1", "succeeded")
|
||||
assert jobs.recover() == 1
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id='retry'"
|
||||
).fetchone()
|
||||
) == ("queued", None, None)
|
||||
assert JobLeases(database).acquire("owner-2", 30) == "retry"
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT attempt, state, lease_owner FROM jobs WHERE job_id='retry'"
|
||||
).fetchone()
|
||||
) == (2, "running", "owner-2")
|
||||
jobs.create("exhaust", "ocr", max_attempts=1)
|
||||
assert jobs.acquire("owner-3", 30) == "exhaust"
|
||||
_expire(database, "exhaust")
|
||||
assert jobs.recover() == 1
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, attempt, lease_owner, lease_expires_at "
|
||||
"FROM jobs WHERE job_id='exhaust'"
|
||||
).fetchone()
|
||||
) == ("failed", 1, None, None)
|
||||
assert jobs.acquire("owner-4", 30) is None
|
||||
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
|
||||
jobs.create(job_id, "ocr")
|
||||
assert jobs.acquire("owner-5", 30) == job_id
|
||||
jobs.finish(job_id, "owner-5", target)
|
||||
with database.connect() as connection:
|
||||
for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")):
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id = ?",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
) == (target, None, None)
|
||||
|
||||
|
||||
def test_retry_wait_finish_rejected_without_changing_lease(database: StorageDatabase) -> None:
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("job-1", "ocr")
|
||||
jobs.acquire("owner-1", 30)
|
||||
with pytest.raises(ValueError):
|
||||
jobs.finish("job-1", "owner-1", "retry_wait")
|
||||
with database.connect() as connection:
|
||||
assert tuple(
|
||||
connection.execute(
|
||||
"SELECT state, lease_owner FROM jobs WHERE job_id='job-1'"
|
||||
).fetchone()
|
||||
) == ("running", "owner-1")
|
||||
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from game_data_capture.pipeline.jobs import JobLeases
|
||||
from game_data_capture.storage.blobs import ContentAddressedBlobStore
|
||||
from game_data_capture.storage.database import StorageDatabase
|
||||
from game_data_capture.storage.repository import CaptureRepository, StorageError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository]:
|
||||
root = (tmp_path / "utility").resolve()
|
||||
migrations = Path(__file__).parents[3] / "migrations"
|
||||
database = StorageDatabase(root, migrations.resolve())
|
||||
database.migrate()
|
||||
return database, CaptureRepository(database, ContentAddressedBlobStore(root))
|
||||
|
||||
|
||||
def test_identical_occurrences_are_deduplicated_and_receipts_survive_restart(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-1"
|
||||
)
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-2", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-2"
|
||||
)
|
||||
# A simulated restart uses fresh repository objects and a repeated intake receipt.
|
||||
restarted = CaptureRepository(database, ContentAddressedBlobStore(database.root))
|
||||
assert (
|
||||
restarted.receive(
|
||||
"capture-other", "receipt-2", [b"same"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-2"
|
||||
)
|
||||
assert restarted.report()["blobs"] == 1
|
||||
assert restarted.report()["captures"] == 2
|
||||
|
||||
|
||||
def test_capture_never_references_missing_blob_and_cas_rejects_stale(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"complete"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone()
|
||||
assert row is not None and repository.blobs.verify(str(row["blob_sha256"])) == len(b"complete")
|
||||
assert repository.transition("capture-1", "queued", 0) == 1
|
||||
with pytest.raises(StorageError):
|
||||
repository.transition("capture-1", "failed", 0)
|
||||
|
||||
|
||||
def test_reprocessing_is_append_only_and_leases_enforce_owner(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {"raw": "one"})
|
||||
repository.append_ocr_run("run-2", "capture-1", "ocr", "2", {"raw": "two"})
|
||||
assert database.connect().execute("SELECT count(*) FROM ocr_runs").fetchone()[0] == 2
|
||||
jobs = JobLeases(database)
|
||||
jobs.create("job-1", "ocr", "capture-1")
|
||||
assert jobs.acquire("worker-a", 30) == "job-1"
|
||||
with pytest.raises(StorageError):
|
||||
jobs.finish("job-1", "worker-b", "succeeded")
|
||||
jobs.finish("job-1", "worker-a", "succeeded")
|
||||
|
||||
|
||||
def test_replayed_receipt_does_not_consume_payload(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
|
||||
consumed = [False]
|
||||
|
||||
class MustNotConsume:
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
consumed[0] = True
|
||||
raise AssertionError("receipt replay consumed payload")
|
||||
|
||||
assert (
|
||||
repository.receive(
|
||||
"capture-2", "receipt-1", MustNotConsume(), media_type="image/png", max_bytes=100
|
||||
)
|
||||
== "capture-1"
|
||||
)
|
||||
assert not consumed[0]
|
||||
|
||||
|
||||
def test_every_provenance_kind_is_insert_only(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
|
||||
repository.append_region("region-1", "run-1", {})
|
||||
repository.append_classification("class-1", "run-1", {})
|
||||
repository.append_candidate("candidate-1", "run-1", {})
|
||||
repository.append_routing("route-1", "candidate-1", {})
|
||||
repository.append_review("review-1", "candidate-1", {})
|
||||
repository.append_lineage("lineage-1", "child-1", "parent-1", "derived")
|
||||
repository.append_export("export-1", "ready", {})
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {})
|
||||
payloads = {
|
||||
"ocr_runs": "payload_json",
|
||||
"regions": "region_json",
|
||||
"classifications": "classification_json",
|
||||
"extraction_candidates": "candidate_json",
|
||||
"routing_decisions": "decision_json",
|
||||
"review_events": "event_json",
|
||||
"exports": "manifest_json",
|
||||
}
|
||||
with database.connect() as connection:
|
||||
for table in (
|
||||
"ocr_runs",
|
||||
"regions",
|
||||
"classifications",
|
||||
"extraction_candidates",
|
||||
"routing_decisions",
|
||||
"review_events",
|
||||
"lineage",
|
||||
"exports",
|
||||
):
|
||||
assert connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0] == 1
|
||||
for table, column in payloads.items():
|
||||
json.loads(connection.execute(f"SELECT {column} FROM {table}").fetchone()[0])
|
||||
assert (
|
||||
connection.execute("SELECT count(*) FROM ocr_runs WHERE run_id='run-1'").fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_failure_sanitization_and_provenance_bounds(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_ocr_run("run", "capture-1", "", "1", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_ocr_run("run", "capture-1", "x" * 101, "1", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_lineage("line", "child", "parent", "")
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_export("export", "", {})
|
||||
with pytest.raises(ValueError):
|
||||
repository.append_candidate("candidate", "run", "x" * 1_000_001)
|
||||
long_token = "x" * 600
|
||||
short_message = (
|
||||
"sk-secret123 Authorization: Bearer auth-secret token=token-secret secret=hidden "
|
||||
"password=pw-secret api_key=key-secret /home/agent/private.png\n"
|
||||
)
|
||||
repository.record_failure("failure-1", "capture", "capture-1", "failure-code", short_message)
|
||||
repository.record_failure("failure-2", "capture", "capture-1", "failure-code", long_token)
|
||||
with database.connect() as connection:
|
||||
rows = list(
|
||||
connection.execute("SELECT code, message FROM runtime_failures ORDER BY failure_id")
|
||||
)
|
||||
assert len(rows) == 2
|
||||
for row in rows:
|
||||
assert len(row["message"]) <= 512 and len(row["code"]) <= 64
|
||||
assert "\n" not in row["message"]
|
||||
for value in (
|
||||
"sk-secret123",
|
||||
"auth-secret",
|
||||
"token-secret",
|
||||
"hidden",
|
||||
"pw-secret",
|
||||
"key-secret",
|
||||
"/home/agent/private.png",
|
||||
"\n",
|
||||
):
|
||||
assert value not in rows[0]["message"]
|
||||
assert long_token not in rows[1]["message"]
|
||||
with pytest.raises(ValueError):
|
||||
repository.record_failure("failure-2", "BAD", "capture-1", "code", "message")
|
||||
|
||||
|
||||
def test_complete_blob_after_capture_insert_failure_is_purgeable_orphan(
|
||||
storage: tuple[StorageDatabase, CaptureRepository],
|
||||
) -> None:
|
||||
database, repository = storage
|
||||
repository.receive(
|
||||
"capture-1", "receipt-1", [b"payload-a"], media_type="image/png", max_bytes=100
|
||||
)
|
||||
payload_b = b"payload-b"
|
||||
digest_b = hashlib.sha256(payload_b).hexdigest()
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
repository.receive(
|
||||
"capture-1", "receipt-2", [payload_b], media_type="image/png", max_bytes=100
|
||||
)
|
||||
assert repository.blobs.verify(digest_b) == len(payload_b)
|
||||
with database.connect() as connection:
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT count(*) FROM blobs WHERE sha256 = ?", (digest_b,)
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest_b,)
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
assert repository.blobs._path(digest_b).relative_to(
|
||||
repository.blobs.root
|
||||
) in repository.blobs.purge_dry_run(set())
|
||||
Reference in New Issue
Block a user