diff --git a/.opencode/agents/gdc-python-core.md b/.opencode/agents/gdc-python-core.md index 4b938d4..e1d0d09 100644 --- a/.opencode/agents/gdc-python-core.md +++ b/.opencode/agents/gdc-python-core.md @@ -1,5 +1,5 @@ --- -description: Implements the game-data-capture Python package, configuration, diagnostics, dependency lock, and approved OCR dependency integration. +description: Implements the game-data-capture Python package, configuration, diagnostics, dependency lock, and approved web and OCR dependency integration. mode: subagent hidden: true model: openai/gpt-5.6-terra @@ -35,11 +35,12 @@ permission: "tools/game-data-capture/tests/unit/test_cli.py": allow "tmp/game-data-capture-plan/handoffs/GDC-002.md": allow "tmp/game-data-capture-plan/handoffs/GDC-007.md": allow + "tmp/game-data-capture-plan/handoffs/GDC-005.md": allow bash: ask task: deny --- -You own Python packaging and dependency integration. The coordinator assigns GDC-002 or the dependency-integration phase of GDC-007. Refuse implementation unless `STATUS.md` marks that task `in_progress`. Read the task and dependency handoffs before editing. +You own Python packaging and dependency integration. The coordinator assigns GDC-002, the dependency-integration phase of GDC-005, or the dependency-integration phase of GDC-007. Refuse implementation unless `STATUS.md` marks that task `in_progress`. Read the task and dependency handoffs before editing. Keep imports side-effect free. Use Python 3.12 through `uv`, XDG runtime paths, typed configuration, redacted logs, and explicit dependency pins. Leave runtime and subsystem wiring to `gdc-runtime-integration`. diff --git a/.opencode/agents/gdc-storage.md b/.opencode/agents/gdc-storage.md index 88097b6..1a81f58 100644 --- a/.opencode/agents/gdc-storage.md +++ b/.opencode/agents/gdc-storage.md @@ -1,5 +1,5 @@ --- -description: Implements utility-local SQLite migrations, content-addressed blobs, durable jobs, provenance, recovery, and safe purge behavior. +description: Implements utility-local SQLite migrations, content-addressed blobs, durable jobs, provenance, recovery, safe purge behavior, and approved GDC-005 corpus persistence. mode: subagent hidden: true model: openai/gpt-5.6-terra @@ -26,12 +26,13 @@ permission: "tools/game-data-capture/tests/unit/storage/**": allow "tools/game-data-capture/tests/integration/storage/**": allow "tmp/game-data-capture-plan/handoffs/GDC-003.md": allow + "tmp/game-data-capture-plan/handoffs/GDC-005.md": allow bash: deny task: deny --- -You own GDC-003 persistence. Refuse implementation unless the coordinator assigns GDC-003, `STATUS.md` marks it `in_progress`, and GDC-001 and GDC-002 passed review. Read the frozen contracts and handoffs before editing. +You own GDC-003 persistence or the approved persistence phase of GDC-005. Refuse implementation unless the coordinator assigns one task, `STATUS.md` marks it `in_progress`, and its dependencies passed review. Read the frozen contracts and dependency handoffs before editing. Keep the utility database separate from SHD.build. Use checksummed additive migrations, foreign keys, WAL, atomic blob installation, compare-and-swap transitions, and recoverable leases. Model capture occurrences separately from blob identity. Test crashes at transaction and filesystem boundaries. -Do not edit domain contracts, pipeline policy, routes, or application database code. Write `handoffs/GDC-003.md` and do not commit or push. +For GDC-005, implement only the reviewed durable corpus-port operations described in its task and handoff. Do not edit domain contracts, pipeline policy, routes, or application database code. Write the assigned task handoff and do not commit or push. diff --git a/tools/game-data-capture/.python-version b/tools/game-data-capture/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/tools/game-data-capture/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/tools/game-data-capture/config.example.toml b/tools/game-data-capture/config.example.toml new file mode 100644 index 0000000..fef317f --- /dev/null +++ b/tools/game-data-capture/config.example.toml @@ -0,0 +1,24 @@ +# Copy this file to $XDG_CONFIG_HOME/shd-capture/config.toml and adjust local limits. +# The utility reads the OpenAI key from the named environment variable only. + +[server] +# Only loopback IP literals are valid: 127.0.0.1 or ::1. +host = "127.0.0.1" +port = 8765 + +[lifecycle] +# One local process owns this instance name and releases its runtime lock on shutdown. +instance_name = "shd-capture" +lock_timeout_seconds = 5 +shutdown_timeout_seconds = 30 + +[limits] +max_image_bytes = 33554432 +max_video_bytes = 4294967296 +max_queue_items = 64 +max_ocr_text_characters = 20000 + +[openai] +# Set this name in the process environment. Do not put the key in this file. +api_key_env = "OPENAI_API_KEY" +model = "gpt-4.1-mini" diff --git a/tools/game-data-capture/contracts/README.md b/tools/game-data-capture/contracts/README.md new file mode 100644 index 0000000..c94bc48 --- /dev/null +++ b/tools/game-data-capture/contracts/README.md @@ -0,0 +1,28 @@ +# Game Data Capture Contracts + +RG-1 freezes contract version 1 for the taxonomy, workflow states, evidence model, archive manifest, and ISAC handoff boundary. + +## Files + +- `domain-contract-v1.json` lists every family/view pair, record type, Gear slot, abstention reason, state, and allowed transition. +- `screenshot-classification.schema.json` validates one hierarchical screen classification. +- `evidence-archive-manifest-v1.schema.json` validates a private evidence archive manifest. +- `isac-handoff-v1.schema.json` validates a bounded draft handoff for authenticated ISAC intake. +- `examples/classification/` contains one valid and one invalid classification for each family, including explicit `unknown` abstention. +- `examples/archive/` and `examples/handoff/` exercise each versioned document schema. + +The schemas use JSON Schema Draft 2020-12, internal `$ref` values, standard formats, and no implementation-specific keywords. Python `jsonschema` and TypeScript validators such as Ajv can compile the files without model conversion. Consumers must reject a document unless `schema_version` equals the schema's `const` value. Consumers must also run semantic reference validation because JSON Schema cannot prove that an ID points to an item in another array. + +## Evidence Rules + +Capture workers store source text, a normalized value, confidence, engine identity, an image digest, a bounded region, and append-only correction history for each evidence value. A null normalized value requires a `missing_reason`. A present normalized value requires a null `missing_reason`. Each correction's `before` state must match the preceding `after` state. Timestamps must increase, and the final `after` state must match the current evidence value. + +Workers record the configured item shown on screen under `observed_instances`, including its visible level, quality, tier, upgrade state, and values. Workers place proposed reusable definitions under `canonical_candidates` and handoff `records`. A worker must not copy an observed roll, relationship, mechanic, range endpoint, or option list into a canonical candidate unless evidence establishes that canonical claim. + +## Trust Boundary + +Archive and handoff documents contain content hashes and logical artifact IDs. They exclude host paths and credentials. The handoff declares `mutation_authority: "none"`. Importing or inspecting it grants no permission and performs no mutation. An authenticated ISAC action must revalidate each draft and preserve contributor moderation and audit behavior. + +Archive images and handoff images allow at most 64 MiB. Private archive videos allow at most 16 GiB, matching the configured capture ceiling. Schema validation enforces media-specific byte limits. Semantic validation enforces unique entity and correction IDs, local references, evidence artifact hashes, correction chains, and normalized image bounds. + +The contracts assign explicit states or reasons to cancellation, resource exhaustion, ambiguous classification, unsupported content, and extraction failure. Recovery requires a listed transition. Unknown transitions and future schema versions fail closed. diff --git a/tools/game-data-capture/contracts/domain-contract-v1.json b/tools/game-data-capture/contracts/domain-contract-v1.json new file mode 100644 index 0000000..fae0b1b --- /dev/null +++ b/tools/game-data-capture/contracts/domain-contract-v1.json @@ -0,0 +1,143 @@ +{ + "contract_version": 1, + "families": { + "weapon": { + "gear_slot": false, + "views": { + "detail": ["weapon", "weapon_talent"], + "blueprint": ["weapon", "weapon_talent"], + "crafting": ["weapon"], + "stat_range": ["weapon"], + "talent_grid": ["weapon_talent"], + "augmentation": ["weapon_augmentation_profile"] + } + }, + "gear": { + "gear_slot": true, + "views": { + "detail": ["gear", "brand_set"], + "blueprint": ["gear", "brand_set"], + "talent": ["gear_talent"], + "augmentation": ["gear_augmentation_profile"] + } + }, + "skill_chip": { + "gear_slot": false, + "views": { + "detail": ["skill_chip_set", "skill_chip_roll_pool"], + "set_bonus": ["skill_chip_set_bonus"], + "talent": ["skill_chip_talent"] + } + }, + "os_protocol": { + "gear_slot": false, + "views": { + "detail": ["os_protocol"], + "secondary_profile": ["os_protocol_secondary_profile"], + "talent": ["os_protocol_talent"] + } + }, + "specialization": { + "gear_slot": false, + "views": { + "overview": ["specialization"], + "details": ["specialization"], + "ability": ["specialization_ability"], + "focus_details": ["specialization_focus"], + "focus_tree": ["specialization_focus", "specialization_tree_node"], + "rank_details": ["specialization_tree_rank"] + } + }, + "unknown": { + "gear_slot": false, + "views": {"unknown": []} + } + }, + "gear_slots": ["Mask", "Gloves", "Backpack", "Body Armor", "Holster", "Knee Pads"], + "abstention_reasons": [ + "low_confidence", + "ambiguous_family", + "ambiguous_view", + "unsupported_family", + "unsupported_view", + "insufficient_visual_evidence", + "occluded", + "corrupt_image", + "resource_limit", + "cancelled" + ], + "missing_reasons": [ + "not_visible", + "occluded", + "unreadable", + "not_present_on_screen", + "unsupported", + "ambiguous", + "extraction_failed", + "not_applicable", + "redacted" + ], + "states": { + "capture": { + "received": ["spooling", "deduplicated", "failed", "cancelled"], + "spooling": ["stored", "failed", "cancelled"], + "stored": ["queued", "failed"], + "deduplicated": [], + "queued": [], + "failed": ["spooling", "cancelled"], + "cancelled": [] + }, + "job": { + "queued": ["running", "cancelled"], + "running": ["succeeded", "retry_wait", "failed", "cancelled"], + "retry_wait": ["queued", "failed", "cancelled"], + "succeeded": [], + "failed": ["queued"], + "cancelled": [] + }, + "extraction": { + "pending": ["local_processing", "cancelled"], + "local_processing": ["candidate_ready", "fallback_pending", "abstained", "failed", "cancelled"], + "fallback_pending": ["fallback_processing", "cancelled"], + "fallback_processing": ["candidate_ready", "abstained", "failed", "cancelled"], + "candidate_ready": [], + "abstained": [], + "failed": ["pending"], + "cancelled": [] + }, + "grouping": { + "ungrouped": ["proposed", "cancelled"], + "proposed": ["grouped", "rejected", "cancelled"], + "grouped": ["split", "superseded"], + "split": ["superseded"], + "superseded": [], + "rejected": ["ungrouped"], + "cancelled": [] + }, + "review": { + "pending": ["in_review", "cancelled"], + "in_review": ["changes_requested", "approved", "rejected", "cancelled"], + "changes_requested": ["in_review", "rejected", "cancelled"], + "approved": ["in_review"], + "rejected": ["in_review"], + "cancelled": [] + }, + "export": { + "not_requested": ["queued"], + "queued": ["building", "cancelled"], + "building": ["ready", "failed", "cancelled"], + "ready": ["expired"], + "failed": ["queued", "cancelled"], + "cancelled": [], + "expired": ["queued"] + }, + "error": { + "open": ["retryable", "terminal", "dismissed"], + "retryable": ["retrying", "terminal", "dismissed"], + "retrying": ["recovered", "retryable", "terminal"], + "recovered": [], + "terminal": ["dismissed"], + "dismissed": [] + } + } +} diff --git a/tools/game-data-capture/contracts/evidence-archive-manifest-v1.schema.json b/tools/game-data-capture/contracts/evidence-archive-manifest-v1.schema.json new file mode 100644 index 0000000..61accce --- /dev/null +++ b/tools/game-data-capture/contracts/evidence-archive-manifest-v1.schema.json @@ -0,0 +1,321 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://shd.build/schemas/game-data-capture/evidence-archive-manifest-v1.json", + "title": "Game data capture evidence archive manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "archive_id", "created_at", "generator", "artifacts", "captures", + "screens", "jobs", "extractions", "groups", "reviews", "exports", "errors", + "evidence_values", "observed_instances", "canonical_candidates" + ], + "properties": { + "schema_version": {"const": 1}, + "archive_id": {"$ref": "#/$defs/id"}, + "created_at": {"$ref": "#/$defs/timestamp"}, + "generator": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "policy_version"], + "properties": { + "name": {"type": "string", "minLength": 1, "maxLength": 100}, + "version": {"type": "string", "minLength": 1, "maxLength": 50}, + "policy_version": {"type": "string", "minLength": 1, "maxLength": 50} + } + }, + "artifacts": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/artifact"}}, + "captures": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/capture"}}, + "screens": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/screen"}}, + "jobs": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/job"}}, + "extractions": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/extraction"}}, + "groups": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/group"}}, + "reviews": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/review"}}, + "exports": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/export"}}, + "errors": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/error"}}, + "evidence_values": {"type": "array", "maxItems": 100000, "items": {"$ref": "#/$defs/evidenceValue"}}, + "observed_instances": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/observedInstance"}}, + "canonical_candidates": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/canonicalCandidate"}} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "timestamp": {"type": "string", "format": "date-time", "maxLength": 40}, + "family": {"enum": ["weapon", "gear", "skill_chip", "os_protocol", "specialization", "unknown"]}, + "recordType": {"enum": [ + "weapon", "weapon_talent", "weapon_augmentation_profile", "gear", "brand_set", + "gear_talent", "gear_augmentation_profile", "skill_chip_set", "skill_chip_set_bonus", + "skill_chip_roll_pool", "skill_chip_talent", "os_protocol", "os_protocol_secondary_profile", + "os_protocol_talent", "specialization", "specialization_focus", "specialization_ability", + "specialization_tree_node", "specialization_tree_rank" + ]}, + "missingReason": {"enum": [ + "not_visible", "occluded", "unreadable", "not_present_on_screen", "unsupported", + "ambiguous", "extraction_failed", "not_applicable", "redacted" + ]}, + "jsonValue": { + "oneOf": [ + {"type": "null"}, {"type": "boolean"}, {"type": "number"}, + {"type": "string", "maxLength": 8192}, + {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/jsonValue"}}, + {"type": "object", "maxProperties": 500, "additionalProperties": {"$ref": "#/$defs/jsonValue"}} + ] + }, + "sourceEngine": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "engine_id", "engine_version"], + "properties": { + "kind": {"enum": ["local_ocr", "openai_vision", "classifier", "manual", "video_decoder"]}, + "engine_id": {"type": "string", "minLength": 1, "maxLength": 100}, + "engine_version": {"type": "string", "minLength": 1, "maxLength": 100} + } + }, + "region": { + "oneOf": [ + { + "type": "object", "additionalProperties": false, + "required": ["coordinate_space", "x", "y", "width", "height"], + "properties": { + "coordinate_space": {"const": "normalized"}, + "x": {"type": "number", "minimum": 0, "maximum": 1}, + "y": {"type": "number", "minimum": 0, "maximum": 1}, + "width": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "height": {"type": "number", "exclusiveMinimum": 0, "maximum": 1} + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["coordinate_space", "x", "y", "width", "height"], + "properties": { + "coordinate_space": {"const": "pixels"}, + "x": {"type": "integer", "minimum": 0, "maximum": 100000}, + "y": {"type": "integer", "minimum": 0, "maximum": 100000}, + "width": {"type": "integer", "minimum": 1, "maximum": 100000}, + "height": {"type": "integer", "minimum": 1, "maximum": 100000} + } + } + ] + }, + "valueState": { + "type": "object", + "additionalProperties": false, + "required": ["raw_text", "normalized_value", "missing_reason"], + "properties": { + "raw_text": {"type": ["string", "null"], "maxLength": 8192}, + "normalized_value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]} + }, + "oneOf": [ + {"properties": {"normalized_value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"normalized_value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "correction": { + "type": "object", "additionalProperties": false, + "required": ["correction_id", "corrected_at", "actor", "reason", "before", "after"], + "properties": { + "correction_id": {"$ref": "#/$defs/id"}, + "corrected_at": {"$ref": "#/$defs/timestamp"}, + "actor": {"enum": ["local_user", "system_reprocess"]}, + "reason": {"type": "string", "minLength": 1, "maxLength": 1000}, + "before": {"$ref": "#/$defs/valueState"}, + "after": {"$ref": "#/$defs/valueState"} + } + }, + "evidenceValue": { + "type": "object", "additionalProperties": false, + "required": [ + "evidence_id", "field_path", "raw_text", "normalized_value", "missing_reason", + "confidence", "source_engine", "image_artifact_id", "image_sha256", "region", "correction_history" + ], + "properties": { + "evidence_id": {"$ref": "#/$defs/id"}, + "field_path": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,256}$"}, + "raw_text": {"type": ["string", "null"], "maxLength": 8192}, + "normalized_value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]}, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "source_engine": {"$ref": "#/$defs/sourceEngine"}, + "image_artifact_id": {"$ref": "#/$defs/id"}, + "image_sha256": {"$ref": "#/$defs/sha256"}, + "region": {"$ref": "#/$defs/region"}, + "correction_history": {"type": "array", "maxItems": 100, "items": {"$ref": "#/$defs/correction"}} + }, + "oneOf": [ + {"properties": {"normalized_value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"normalized_value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "fieldClaim": { + "type": "object", "additionalProperties": false, + "required": ["path", "value", "missing_reason", "evidence_refs", "uncertainty_note"], + "properties": { + "path": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,256}$"}, + "value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]}, + "evidence_refs": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "uncertainty_note": {"type": ["string", "null"], "maxLength": 2000} + }, + "oneOf": [ + {"properties": {"value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "classification": { + "type": "object", "additionalProperties": false, + "required": ["family", "view", "gear_slot", "record_types", "confidence", "abstention_reason"], + "properties": { + "family": {"$ref": "#/$defs/family"}, "view": {"type": "string"}, + "gear_slot": {"type": ["string", "null"]}, + "record_types": {"type": "array", "uniqueItems": true, "maxItems": 8, "items": {"$ref": "#/$defs/recordType"}}, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "abstention_reason": {"type": ["string", "null"]} + }, + "oneOf": [ + {"properties": {"family": {"const": "weapon"}, "view": {"enum": ["detail", "blueprint", "crafting", "stat_range", "talent_grid", "augmentation"]}, "gear_slot": {"const": null}, "record_types": {"minItems": 1, "items": {"enum": ["weapon", "weapon_talent", "weapon_augmentation_profile"]}}, "abstention_reason": {"const": null}}}, + {"properties": {"family": {"const": "gear"}, "view": {"enum": ["detail", "blueprint", "talent", "augmentation"]}, "gear_slot": {"enum": ["Mask", "Gloves", "Backpack", "Body Armor", "Holster", "Knee Pads"]}, "record_types": {"minItems": 1, "items": {"enum": ["gear", "brand_set", "gear_talent", "gear_augmentation_profile"]}}, "abstention_reason": {"const": null}}}, + {"properties": {"family": {"const": "skill_chip"}, "view": {"enum": ["detail", "set_bonus", "talent"]}, "gear_slot": {"const": null}, "record_types": {"minItems": 1, "items": {"enum": ["skill_chip_set", "skill_chip_set_bonus", "skill_chip_roll_pool", "skill_chip_talent"]}}, "abstention_reason": {"const": null}}}, + {"properties": {"family": {"const": "os_protocol"}, "view": {"enum": ["detail", "secondary_profile", "talent"]}, "gear_slot": {"const": null}, "record_types": {"minItems": 1, "items": {"enum": ["os_protocol", "os_protocol_secondary_profile", "os_protocol_talent"]}}, "abstention_reason": {"const": null}}}, + {"properties": {"family": {"const": "specialization"}, "view": {"enum": ["overview", "details", "ability", "focus_details", "focus_tree", "rank_details"]}, "gear_slot": {"const": null}, "record_types": {"minItems": 1, "items": {"enum": ["specialization", "specialization_focus", "specialization_ability", "specialization_tree_node", "specialization_tree_rank"]}}, "abstention_reason": {"const": null}}}, + {"properties": {"family": {"const": "unknown"}, "view": {"const": "unknown"}, "gear_slot": {"const": null}, "record_types": {"maxItems": 0}, "abstention_reason": {"enum": ["low_confidence", "ambiguous_family", "ambiguous_view", "unsupported_family", "unsupported_view", "insufficient_visual_evidence", "occluded", "corrupt_image", "resource_limit", "cancelled"]}}} + ] + }, + "artifact": { + "type": "object", "additionalProperties": false, + "required": ["artifact_id", "sha256", "media_type", "byte_size", "sensitivity", "parent_artifact_id", "video_timestamp_ms"], + "properties": { + "artifact_id": {"$ref": "#/$defs/id"}, "sha256": {"$ref": "#/$defs/sha256"}, + "media_type": {"enum": ["image/png", "image/webp", "image/avif", "video/mp4", "video/webm", "video/x-matroska"]}, + "byte_size": {"type": "integer", "minimum": 1}, + "sensitivity": {"const": "private"}, + "parent_artifact_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "video_timestamp_ms": {"type": ["integer", "null"], "minimum": 0, "maximum": 86400000} + }, + "oneOf": [ + { + "properties": { + "media_type": {"enum": ["image/png", "image/webp", "image/avif"]}, + "byte_size": {"maximum": 67108864} + } + }, + { + "properties": { + "media_type": {"enum": ["video/mp4", "video/webm", "video/x-matroska"]}, + "byte_size": {"maximum": 17179869184} + } + } + ] + }, + "capture": { + "type": "object", "additionalProperties": false, + "required": ["capture_id", "artifact_id", "state", "received_at", "failure_reason"], + "properties": { + "capture_id": {"$ref": "#/$defs/id"}, "artifact_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["received", "spooling", "stored", "deduplicated", "queued", "failed", "cancelled"]}, + "received_at": {"$ref": "#/$defs/timestamp"}, + "failure_reason": {"type": ["string", "null"], "maxLength": 2000} + } + }, + "screen": { + "type": "object", "additionalProperties": false, + "required": ["screen_id", "artifact_id", "classified_at", "classification"], + "properties": { + "screen_id": {"$ref": "#/$defs/id"}, "artifact_id": {"$ref": "#/$defs/id"}, + "classified_at": {"$ref": "#/$defs/timestamp"}, "classification": {"$ref": "#/$defs/classification"} + } + }, + "job": { + "type": "object", "additionalProperties": false, + "required": ["job_id", "kind", "state", "attempt", "max_attempts", "last_error_id"], + "properties": { + "job_id": {"$ref": "#/$defs/id"}, + "kind": {"enum": ["classify", "ocr", "vision_fallback", "frame_extract", "export"]}, + "state": {"enum": ["queued", "running", "retry_wait", "succeeded", "failed", "cancelled"]}, + "attempt": {"type": "integer", "minimum": 0, "maximum": 20}, + "max_attempts": {"type": "integer", "minimum": 1, "maximum": 20}, + "last_error_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "extraction": { + "type": "object", "additionalProperties": false, + "required": ["extraction_id", "screen_id", "state", "evidence_refs", "abstention_reason"], + "properties": { + "extraction_id": {"$ref": "#/$defs/id"}, "screen_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["pending", "local_processing", "fallback_pending", "fallback_processing", "candidate_ready", "abstained", "failed", "cancelled"]}, + "evidence_refs": {"type": "array", "maxItems": 1000, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "abstention_reason": {"type": ["string", "null"], "maxLength": 100} + } + }, + "group": { + "type": "object", "additionalProperties": false, + "required": ["group_id", "state", "screen_ids", "supersedes_group_ids"], + "properties": { + "group_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["ungrouped", "proposed", "grouped", "split", "superseded", "rejected", "cancelled"]}, + "screen_ids": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "supersedes_group_ids": {"type": "array", "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}} + } + }, + "review": { + "type": "object", "additionalProperties": false, + "required": ["review_id", "candidate_id", "state", "updated_at", "note"], + "properties": { + "review_id": {"$ref": "#/$defs/id"}, "candidate_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["pending", "in_review", "changes_requested", "approved", "rejected", "cancelled"]}, + "updated_at": {"$ref": "#/$defs/timestamp"}, "note": {"type": ["string", "null"], "maxLength": 4000} + } + }, + "export": { + "type": "object", "additionalProperties": false, + "required": ["export_id", "state", "artifact_id", "failure_reason"], + "properties": { + "export_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["not_requested", "queued", "building", "ready", "failed", "cancelled", "expired"]}, + "artifact_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "failure_reason": {"type": ["string", "null"], "maxLength": 2000} + } + }, + "error": { + "type": "object", "additionalProperties": false, + "required": ["error_id", "state", "code", "message", "retry_after"], + "properties": { + "error_id": {"$ref": "#/$defs/id"}, + "state": {"enum": ["open", "retryable", "retrying", "recovered", "terminal", "dismissed"]}, + "code": {"enum": ["cancelled", "resource_limit", "timeout", "invalid_input", "engine_failure", "storage_failure", "network_failure", "schema_rejected", "unsupported", "ambiguous"]}, + "message": {"type": "string", "minLength": 1, "maxLength": 2000}, + "retry_after": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/timestamp"}]} + } + }, + "observedContext": { + "type": "object", "additionalProperties": false, + "required": ["level", "quality", "tier", "upgrade_state"], + "properties": { + "level": {"$ref": "#/$defs/fieldClaim"}, "quality": {"$ref": "#/$defs/fieldClaim"}, + "tier": {"$ref": "#/$defs/fieldClaim"}, "upgrade_state": {"$ref": "#/$defs/fieldClaim"} + } + }, + "observedInstance": { + "type": "object", "additionalProperties": false, + "required": ["instance_id", "family", "source_screen_ids", "observed_context", "fields"], + "properties": { + "instance_id": {"$ref": "#/$defs/id"}, "family": {"$ref": "#/$defs/family"}, + "source_screen_ids": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "observed_context": {"$ref": "#/$defs/observedContext"}, + "fields": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/fieldClaim"}} + } + }, + "canonicalCandidate": { + "type": "object", "additionalProperties": false, + "required": ["candidate_id", "record_type", "source_instance_ids", "evidence_refs", "fields", "completeness", "relationship_status", "mechanic_status"], + "properties": { + "candidate_id": {"$ref": "#/$defs/id"}, "record_type": {"$ref": "#/$defs/recordType"}, + "source_instance_ids": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "evidence_refs": {"type": "array", "minItems": 1, "maxItems": 1000, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "fields": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/fieldClaim"}}, + "completeness": {"enum": ["complete", "incomplete", "unsupported"]}, + "relationship_status": {"enum": ["evidenced", "unresolved", "unsupported", "not_applicable"]}, + "mechanic_status": {"enum": ["evidenced", "unresolved", "unsupported", "not_applicable"]} + } + } + } +} diff --git a/tools/game-data-capture/contracts/examples/archive/manifest.invalid.json b/tools/game-data-capture/contracts/examples/archive/manifest.invalid.json new file mode 100644 index 0000000..3f33203 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/archive/manifest.invalid.json @@ -0,0 +1,4 @@ +{ + "schema_version": 2, + "archive_id": "future-archive" +} diff --git a/tools/game-data-capture/contracts/examples/archive/manifest.valid.json b/tools/game-data-capture/contracts/examples/archive/manifest.valid.json new file mode 100644 index 0000000..3db9b07 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/archive/manifest.valid.json @@ -0,0 +1,162 @@ +{ + "schema_version": 1, + "archive_id": "archive-001", + "created_at": "2026-07-31T12:00:00Z", + "generator": {"name": "game-data-capture", "version": "0.1.0", "policy_version": "1"}, + "artifacts": [ + { + "artifact_id": "image-001", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "image/png", + "byte_size": 2048, + "sensitivity": "private", + "parent_artifact_id": null, + "video_timestamp_ms": null + } + ], + "captures": [ + {"capture_id": "capture-001", "artifact_id": "image-001", "state": "queued", "received_at": "2026-07-31T11:59:55Z", "failure_reason": null} + ], + "screens": [ + { + "screen_id": "screen-001", + "artifact_id": "image-001", + "classified_at": "2026-07-31T11:59:57Z", + "classification": { + "family": "weapon", + "view": "detail", + "gear_slot": null, + "record_types": ["weapon", "weapon_talent"], + "confidence": 0.97, + "abstention_reason": null + } + } + ], + "jobs": [], + "extractions": [], + "groups": [], + "reviews": [ + {"review_id": "review-001", "candidate_id": "candidate-001", "state": "approved", "updated_at": "2026-07-31T12:00:00Z", "note": "Name verified against the image."} + ], + "exports": [], + "errors": [], + "evidence_values": [ + { + "evidence_id": "evidence-name", + "field_path": "weapon.name", + "raw_text": "LVOA-C", + "normalized_value": "LVOA-C", + "missing_reason": null, + "confidence": 0.99, + "source_engine": {"kind": "local_ocr", "engine_id": "fixture-ocr", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "normalized", "x": 0.1, "y": 0.1, "width": 0.3, "height": 0.08}, + "correction_history": [] + }, + { + "evidence_id": "evidence-level", + "field_path": "observed.level", + "raw_text": "20", + "normalized_value": 20, + "missing_reason": null, + "confidence": 0.98, + "source_engine": {"kind": "local_ocr", "engine_id": "fixture-ocr", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "pixels", "x": 50, "y": 50, "width": 80, "height": 30}, + "correction_history": [] + }, + { + "evidence_id": "evidence-quality", + "field_path": "observed.quality", + "raw_text": "High-end", + "normalized_value": "High-end", + "missing_reason": null, + "confidence": 0.96, + "source_engine": {"kind": "local_ocr", "engine_id": "fixture-ocr", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "pixels", "x": 50, "y": 85, "width": 140, "height": 30}, + "correction_history": [] + }, + { + "evidence_id": "evidence-tier", + "field_path": "observed.tier", + "raw_text": "T1", + "normalized_value": 1, + "missing_reason": null, + "confidence": 0.97, + "source_engine": {"kind": "local_ocr", "engine_id": "fixture-ocr", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "pixels", "x": 200, "y": 85, "width": 60, "height": 30}, + "correction_history": [] + }, + { + "evidence_id": "evidence-upgrade", + "field_path": "observed.upgrade_state", + "raw_text": null, + "normalized_value": null, + "missing_reason": "not_visible", + "confidence": 0, + "source_engine": {"kind": "manual", "engine_id": "review", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "normalized", "x": 0, "y": 0, "width": 1, "height": 1}, + "correction_history": [] + }, + { + "evidence_id": "evidence-damage", + "field_path": "observed.damage", + "raw_text": "12,400", + "normalized_value": 12400, + "missing_reason": null, + "confidence": 1, + "source_engine": {"kind": "manual", "engine_id": "review", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "pixels", "x": 300, "y": 200, "width": 160, "height": 40}, + "correction_history": [ + { + "correction_id": "correction-001", + "corrected_at": "2026-07-31T12:00:00Z", + "actor": "local_user", + "reason": "Corrected a missing zero.", + "before": {"raw_text": "1,240", "normalized_value": 1240, "missing_reason": null}, + "after": {"raw_text": "12,400", "normalized_value": 12400, "missing_reason": null} + } + ] + } + ], + "observed_instances": [ + { + "instance_id": "instance-001", + "family": "weapon", + "source_screen_ids": ["screen-001"], + "observed_context": { + "level": {"path": "observed.level", "value": 20, "missing_reason": null, "evidence_refs": ["evidence-level"], "uncertainty_note": null}, + "quality": {"path": "observed.quality", "value": "High-end", "missing_reason": null, "evidence_refs": ["evidence-quality"], "uncertainty_note": null}, + "tier": {"path": "observed.tier", "value": 1, "missing_reason": null, "evidence_refs": ["evidence-tier"], "uncertainty_note": null}, + "upgrade_state": {"path": "observed.upgrade_state", "value": null, "missing_reason": "not_visible", "evidence_refs": ["evidence-upgrade"], "uncertainty_note": "The screen does not show an upgrade indicator."} + }, + "fields": [ + {"path": "observed.damage", "value": 12400, "missing_reason": null, "evidence_refs": ["evidence-damage"], "uncertainty_note": "This is the configured level-20 instance value, not a canonical range endpoint."} + ] + } + ], + "canonical_candidates": [ + { + "candidate_id": "candidate-001", + "record_type": "weapon", + "source_instance_ids": ["instance-001"], + "evidence_refs": ["evidence-name"], + "fields": [ + {"path": "weapon.name", "value": "LVOA-C", "missing_reason": null, "evidence_refs": ["evidence-name"], "uncertainty_note": null} + ], + "completeness": "incomplete", + "relationship_status": "unresolved", + "mechanic_status": "not_applicable" + } + ] +} diff --git a/tools/game-data-capture/contracts/examples/classification/gear.invalid.json b/tools/game-data-capture/contracts/examples/classification/gear.invalid.json new file mode 100644 index 0000000..ebf8261 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/gear.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "gear", + "view": "detail", + "gear_slot": "Vest", + "record_types": ["gear"], + "confidence": 0.95, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/gear.valid.json b/tools/game-data-capture/contracts/examples/classification/gear.valid.json new file mode 100644 index 0000000..82642e5 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/gear.valid.json @@ -0,0 +1,8 @@ +{ + "family": "gear", + "view": "talent", + "gear_slot": "Body Armor", + "record_types": ["gear_talent"], + "confidence": 0.95, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/os_protocol.invalid.json b/tools/game-data-capture/contracts/examples/classification/os_protocol.invalid.json new file mode 100644 index 0000000..69be12d --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/os_protocol.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "os_protocol", + "view": "secondary_profile", + "gear_slot": null, + "record_types": ["os_protocol_secondary_profile"], + "confidence": 1.3, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/os_protocol.valid.json b/tools/game-data-capture/contracts/examples/classification/os_protocol.valid.json new file mode 100644 index 0000000..2d4e242 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/os_protocol.valid.json @@ -0,0 +1,8 @@ +{ + "family": "os_protocol", + "view": "secondary_profile", + "gear_slot": null, + "record_types": ["os_protocol_secondary_profile"], + "confidence": 0.93, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/skill_chip.invalid.json b/tools/game-data-capture/contracts/examples/classification/skill_chip.invalid.json new file mode 100644 index 0000000..7dcaae5 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/skill_chip.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "skill_chip", + "view": "detail", + "gear_slot": null, + "record_types": ["weapon"], + "confidence": 0.94, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/skill_chip.valid.json b/tools/game-data-capture/contracts/examples/classification/skill_chip.valid.json new file mode 100644 index 0000000..ba02519 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/skill_chip.valid.json @@ -0,0 +1,8 @@ +{ + "family": "skill_chip", + "view": "set_bonus", + "gear_slot": null, + "record_types": ["skill_chip_set_bonus"], + "confidence": 0.94, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/specialization.invalid.json b/tools/game-data-capture/contracts/examples/classification/specialization.invalid.json new file mode 100644 index 0000000..990edc3 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/specialization.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "specialization", + "view": "focus_tree", + "gear_slot": null, + "record_types": ["specialization_tree_node"], + "confidence": 0.91, + "abstention_reason": "ambiguous_view" +} diff --git a/tools/game-data-capture/contracts/examples/classification/specialization.valid.json b/tools/game-data-capture/contracts/examples/classification/specialization.valid.json new file mode 100644 index 0000000..46deb0b --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/specialization.valid.json @@ -0,0 +1,8 @@ +{ + "family": "specialization", + "view": "focus_tree", + "gear_slot": null, + "record_types": ["specialization_focus", "specialization_tree_node"], + "confidence": 0.91, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/unknown.invalid.json b/tools/game-data-capture/contracts/examples/classification/unknown.invalid.json new file mode 100644 index 0000000..a5ff7d2 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/unknown.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "unknown", + "view": "unknown", + "gear_slot": null, + "record_types": ["weapon"], + "confidence": 0.22, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/unknown.valid.json b/tools/game-data-capture/contracts/examples/classification/unknown.valid.json new file mode 100644 index 0000000..9837260 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/unknown.valid.json @@ -0,0 +1,8 @@ +{ + "family": "unknown", + "view": "unknown", + "gear_slot": null, + "record_types": [], + "confidence": 0.22, + "abstention_reason": "low_confidence" +} diff --git a/tools/game-data-capture/contracts/examples/classification/weapon.invalid.json b/tools/game-data-capture/contracts/examples/classification/weapon.invalid.json new file mode 100644 index 0000000..851ece3 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/weapon.invalid.json @@ -0,0 +1,8 @@ +{ + "family": "weapon", + "view": "talent", + "gear_slot": null, + "record_types": ["weapon"], + "confidence": 0.9, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/classification/weapon.valid.json b/tools/game-data-capture/contracts/examples/classification/weapon.valid.json new file mode 100644 index 0000000..2cbe504 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/classification/weapon.valid.json @@ -0,0 +1,8 @@ +{ + "family": "weapon", + "view": "detail", + "gear_slot": null, + "record_types": ["weapon", "weapon_talent"], + "confidence": 0.97, + "abstention_reason": null +} diff --git a/tools/game-data-capture/contracts/examples/handoff/isac-handoff.invalid.json b/tools/game-data-capture/contracts/examples/handoff/isac-handoff.invalid.json new file mode 100644 index 0000000..2bc9948 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/handoff/isac-handoff.invalid.json @@ -0,0 +1,4 @@ +{ + "schema_version": 2, + "handoff_id": "future-handoff" +} diff --git a/tools/game-data-capture/contracts/examples/handoff/isac-handoff.valid.json b/tools/game-data-capture/contracts/examples/handoff/isac-handoff.valid.json new file mode 100644 index 0000000..9c7bc17 --- /dev/null +++ b/tools/game-data-capture/contracts/examples/handoff/isac-handoff.valid.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "handoff_id": "handoff-001", + "created_at": "2026-07-31T12:05:00Z", + "source_archive": { + "archive_id": "archive-001", + "manifest_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "schema_version": 1 + }, + "trust_boundary": { + "intended_destination": "authenticated_isac_intake", + "mutation_authority": "none", + "requires_authenticated_action": true, + "preserves_moderation": true, + "creates_audit_events": false + }, + "source_artifacts": [ + { + "artifact_id": "image-001", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "image/png", + "byte_size": 2048 + } + ], + "evidence_values": [ + { + "evidence_id": "evidence-name", + "field_path": "weapon.name", + "raw_text": "LVOA-C", + "normalized_value": "LVOA-C", + "missing_reason": null, + "confidence": 0.99, + "source_engine": {"kind": "local_ocr", "engine_id": "fixture-ocr", "engine_version": "1"}, + "image_artifact_id": "image-001", + "image_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": {"coordinate_space": "normalized", "x": 0.1, "y": 0.1, "width": 0.3, "height": 0.08}, + "correction_history": [] + } + ], + "records": [ + { + "handoff_record_id": "handoff-record-001", + "record_type": "weapon", + "intent": "create_draft", + "target_id": null, + "review_state": "approved", + "source_candidate_ids": ["candidate-001"], + "observed_instance_ids": ["instance-001"], + "fields": [ + {"path": "weapon.name", "value": "LVOA-C", "missing_reason": null, "evidence_refs": ["evidence-name"], "uncertainty_note": null} + ], + "relationships": {"status": "unresolved", "evidence_refs": [], "note": "ISAC must resolve and validate canonical relationships before submission."}, + "mechanics": {"status": "not_applicable", "evidence_refs": [], "note": null} + } + ] +} diff --git a/tools/game-data-capture/contracts/isac-handoff-v1.schema.json b/tools/game-data-capture/contracts/isac-handoff-v1.schema.json new file mode 100644 index 0000000..8ce6b9a --- /dev/null +++ b/tools/game-data-capture/contracts/isac-handoff-v1.schema.json @@ -0,0 +1,196 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://shd.build/schemas/game-data-capture/isac-handoff-v1.json", + "title": "Authenticated ISAC draft handoff v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "handoff_id", "created_at", "source_archive", "trust_boundary", + "source_artifacts", "evidence_values", "records" + ], + "properties": { + "schema_version": {"const": 1}, + "handoff_id": {"$ref": "#/$defs/id"}, + "created_at": {"$ref": "#/$defs/timestamp"}, + "source_archive": { + "type": "object", "additionalProperties": false, + "required": ["archive_id", "manifest_sha256", "schema_version"], + "properties": { + "archive_id": {"$ref": "#/$defs/id"}, "manifest_sha256": {"$ref": "#/$defs/sha256"}, + "schema_version": {"const": 1} + } + }, + "trust_boundary": { + "type": "object", "additionalProperties": false, + "required": ["intended_destination", "mutation_authority", "requires_authenticated_action", "preserves_moderation", "creates_audit_events"], + "properties": { + "intended_destination": {"const": "authenticated_isac_intake"}, + "mutation_authority": {"const": "none"}, + "requires_authenticated_action": {"const": true}, + "preserves_moderation": {"const": true}, + "creates_audit_events": {"const": false} + } + }, + "source_artifacts": {"type": "array", "maxItems": 10000, "items": {"$ref": "#/$defs/sourceArtifact"}}, + "evidence_values": {"type": "array", "maxItems": 100000, "items": {"$ref": "#/$defs/evidenceValue"}}, + "records": {"type": "array", "maxItems": 20000, "items": {"$ref": "#/$defs/handoffRecord"}} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "timestamp": {"type": "string", "format": "date-time", "maxLength": 40}, + "recordType": {"enum": [ + "weapon", "weapon_talent", "weapon_augmentation_profile", "gear", "brand_set", + "gear_talent", "gear_augmentation_profile", "skill_chip_set", "skill_chip_set_bonus", + "skill_chip_roll_pool", "skill_chip_talent", "os_protocol", "os_protocol_secondary_profile", + "os_protocol_talent", "specialization", "specialization_focus", "specialization_ability", + "specialization_tree_node", "specialization_tree_rank" + ]}, + "missingReason": {"enum": [ + "not_visible", "occluded", "unreadable", "not_present_on_screen", "unsupported", + "ambiguous", "extraction_failed", "not_applicable", "redacted" + ]}, + "jsonValue": { + "oneOf": [ + {"type": "null"}, {"type": "boolean"}, {"type": "number"}, + {"type": "string", "maxLength": 8192}, + {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/jsonValue"}}, + {"type": "object", "maxProperties": 500, "additionalProperties": {"$ref": "#/$defs/jsonValue"}} + ] + }, + "sourceEngine": { + "type": "object", "additionalProperties": false, + "required": ["kind", "engine_id", "engine_version"], + "properties": { + "kind": {"enum": ["local_ocr", "openai_vision", "classifier", "manual", "video_decoder"]}, + "engine_id": {"type": "string", "minLength": 1, "maxLength": 100}, + "engine_version": {"type": "string", "minLength": 1, "maxLength": 100} + } + }, + "region": { + "oneOf": [ + { + "type": "object", "additionalProperties": false, + "required": ["coordinate_space", "x", "y", "width", "height"], + "properties": { + "coordinate_space": {"const": "normalized"}, + "x": {"type": "number", "minimum": 0, "maximum": 1}, + "y": {"type": "number", "minimum": 0, "maximum": 1}, + "width": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "height": {"type": "number", "exclusiveMinimum": 0, "maximum": 1} + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["coordinate_space", "x", "y", "width", "height"], + "properties": { + "coordinate_space": {"const": "pixels"}, + "x": {"type": "integer", "minimum": 0, "maximum": 100000}, + "y": {"type": "integer", "minimum": 0, "maximum": 100000}, + "width": {"type": "integer", "minimum": 1, "maximum": 100000}, + "height": {"type": "integer", "minimum": 1, "maximum": 100000} + } + } + ] + }, + "valueState": { + "type": "object", "additionalProperties": false, + "required": ["raw_text", "normalized_value", "missing_reason"], + "properties": { + "raw_text": {"type": ["string", "null"], "maxLength": 8192}, + "normalized_value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]} + }, + "oneOf": [ + {"properties": {"normalized_value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"normalized_value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "correction": { + "type": "object", "additionalProperties": false, + "required": ["correction_id", "corrected_at", "actor", "reason", "before", "after"], + "properties": { + "correction_id": {"$ref": "#/$defs/id"}, "corrected_at": {"$ref": "#/$defs/timestamp"}, + "actor": {"enum": ["local_user", "system_reprocess"]}, + "reason": {"type": "string", "minLength": 1, "maxLength": 1000}, + "before": {"$ref": "#/$defs/valueState"}, "after": {"$ref": "#/$defs/valueState"} + } + }, + "evidenceValue": { + "type": "object", "additionalProperties": false, + "required": ["evidence_id", "field_path", "raw_text", "normalized_value", "missing_reason", "confidence", "source_engine", "image_artifact_id", "image_sha256", "region", "correction_history"], + "properties": { + "evidence_id": {"$ref": "#/$defs/id"}, + "field_path": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,256}$"}, + "raw_text": {"type": ["string", "null"], "maxLength": 8192}, + "normalized_value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]}, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "source_engine": {"$ref": "#/$defs/sourceEngine"}, + "image_artifact_id": {"$ref": "#/$defs/id"}, "image_sha256": {"$ref": "#/$defs/sha256"}, + "region": {"$ref": "#/$defs/region"}, + "correction_history": {"type": "array", "maxItems": 100, "items": {"$ref": "#/$defs/correction"}} + }, + "oneOf": [ + {"properties": {"normalized_value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"normalized_value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "fieldClaim": { + "type": "object", "additionalProperties": false, + "required": ["path", "value", "missing_reason", "evidence_refs", "uncertainty_note"], + "properties": { + "path": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,256}$"}, + "value": {"$ref": "#/$defs/jsonValue"}, + "missing_reason": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/missingReason"}]}, + "evidence_refs": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "uncertainty_note": {"type": ["string", "null"], "maxLength": 2000} + }, + "oneOf": [ + {"properties": {"value": {"type": "null"}, "missing_reason": {"$ref": "#/$defs/missingReason"}}}, + {"properties": {"value": {"not": {"type": "null"}}, "missing_reason": {"type": "null"}}} + ] + }, + "sourceArtifact": { + "type": "object", "additionalProperties": false, + "required": ["artifact_id", "sha256", "media_type", "byte_size"], + "properties": { + "artifact_id": {"$ref": "#/$defs/id"}, "sha256": {"$ref": "#/$defs/sha256"}, + "media_type": {"enum": ["image/png", "image/webp", "image/avif"]}, + "byte_size": {"type": "integer", "minimum": 1, "maximum": 67108864} + } + }, + "evidenceSection": { + "type": "object", "additionalProperties": false, + "required": ["status", "evidence_refs", "note"], + "properties": { + "status": {"enum": ["evidenced", "unresolved", "unsupported", "not_applicable"]}, + "evidence_refs": {"type": "array", "maxItems": 1000, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "note": {"type": ["string", "null"], "maxLength": 4000} + } + }, + "handoffRecord": { + "type": "object", "additionalProperties": false, + "required": [ + "handoff_record_id", "record_type", "intent", "target_id", "review_state", + "source_candidate_ids", "observed_instance_ids", "fields", "relationships", "mechanics" + ], + "properties": { + "handoff_record_id": {"$ref": "#/$defs/id"}, "record_type": {"$ref": "#/$defs/recordType"}, + "intent": {"enum": ["create_draft", "update_draft", "abstain"]}, + "target_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "review_state": {"enum": ["approved", "abstained"]}, + "source_candidate_ids": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "observed_instance_ids": {"type": "array", "minItems": 1, "maxItems": 100, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "fields": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/fieldClaim"}}, + "relationships": {"$ref": "#/$defs/evidenceSection"}, + "mechanics": {"$ref": "#/$defs/evidenceSection"} + }, + "oneOf": [ + {"properties": {"intent": {"const": "create_draft"}, "target_id": {"const": null}, "review_state": {"const": "approved"}}}, + {"properties": {"intent": {"const": "update_draft"}, "target_id": {"$ref": "#/$defs/id"}, "review_state": {"const": "approved"}}}, + {"properties": {"intent": {"const": "abstain"}, "target_id": {"const": null}, "review_state": {"const": "abstained"}, "fields": {"maxItems": 0}}} + ] + } + } +} diff --git a/tools/game-data-capture/contracts/screenshot-classification.schema.json b/tools/game-data-capture/contracts/screenshot-classification.schema.json new file mode 100644 index 0000000..20281c5 --- /dev/null +++ b/tools/game-data-capture/contracts/screenshot-classification.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://shd.build/schemas/game-data-capture/screenshot-classification-v1.json", + "title": "Screenshot classification v1", + "type": "object", + "additionalProperties": false, + "required": ["family", "view", "gear_slot", "record_types", "confidence", "abstention_reason"], + "properties": { + "family": {"enum": ["weapon", "gear", "skill_chip", "os_protocol", "specialization", "unknown"]}, + "view": {"type": "string"}, + "gear_slot": {"type": ["string", "null"]}, + "record_types": {"type": "array", "uniqueItems": true, "maxItems": 8, "items": {"type": "string"}}, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "abstention_reason": {"type": ["string", "null"]} + }, + "oneOf": [ + { + "properties": { + "family": {"const": "weapon"}, + "view": {"enum": ["detail", "blueprint", "crafting", "stat_range", "talent_grid", "augmentation"]}, + "gear_slot": {"const": null}, + "record_types": {"minItems": 1, "items": {"enum": ["weapon", "weapon_talent", "weapon_augmentation_profile"]}}, + "abstention_reason": {"const": null} + } + }, + { + "properties": { + "family": {"const": "gear"}, + "view": {"enum": ["detail", "blueprint", "talent", "augmentation"]}, + "gear_slot": {"enum": ["Mask", "Gloves", "Backpack", "Body Armor", "Holster", "Knee Pads"]}, + "record_types": {"minItems": 1, "items": {"enum": ["gear", "brand_set", "gear_talent", "gear_augmentation_profile"]}}, + "abstention_reason": {"const": null} + } + }, + { + "properties": { + "family": {"const": "skill_chip"}, + "view": {"enum": ["detail", "set_bonus", "talent"]}, + "gear_slot": {"const": null}, + "record_types": {"minItems": 1, "items": {"enum": ["skill_chip_set", "skill_chip_set_bonus", "skill_chip_roll_pool", "skill_chip_talent"]}}, + "abstention_reason": {"const": null} + } + }, + { + "properties": { + "family": {"const": "os_protocol"}, + "view": {"enum": ["detail", "secondary_profile", "talent"]}, + "gear_slot": {"const": null}, + "record_types": {"minItems": 1, "items": {"enum": ["os_protocol", "os_protocol_secondary_profile", "os_protocol_talent"]}}, + "abstention_reason": {"const": null} + } + }, + { + "properties": { + "family": {"const": "specialization"}, + "view": {"enum": ["overview", "details", "ability", "focus_details", "focus_tree", "rank_details"]}, + "gear_slot": {"const": null}, + "record_types": {"minItems": 1, "items": {"enum": ["specialization", "specialization_focus", "specialization_ability", "specialization_tree_node", "specialization_tree_rank"]}}, + "abstention_reason": {"const": null} + } + }, + { + "properties": { + "family": {"const": "unknown"}, + "view": {"const": "unknown"}, + "gear_slot": {"const": null}, + "record_types": {"maxItems": 0}, + "abstention_reason": {"enum": ["low_confidence", "ambiguous_family", "ambiguous_view", "unsupported_family", "unsupported_view", "insufficient_visual_evidence", "occluded", "corrupt_image", "resource_limit", "cancelled"]} + } + } + ] +} diff --git a/tools/game-data-capture/migrations/001_initial.sql b/tools/game-data-capture/migrations/001_initial.sql new file mode 100644 index 0000000..1f8ae0d --- /dev/null +++ b/tools/game-data-capture/migrations/001_initial.sql @@ -0,0 +1,69 @@ +-- Additive utility-local schema. Migration content is fingerprinted at application time. +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + filename TEXT NOT NULL UNIQUE, + checksum TEXT NOT NULL, + applied_at TEXT NOT NULL +); +CREATE TABLE blobs ( + sha256 TEXT PRIMARY KEY CHECK(length(sha256) = 64), + byte_size INTEGER NOT NULL CHECK(byte_size > 0), + media_type TEXT NOT NULL CHECK(length(media_type) <= 100), + created_at TEXT NOT NULL +); +CREATE TABLE captures ( + capture_id TEXT PRIMARY KEY, + blob_sha256 TEXT NOT NULL REFERENCES blobs(sha256) ON DELETE RESTRICT, + state TEXT NOT NULL, + received_at TEXT NOT NULL, + receipt_token TEXT NOT NULL UNIQUE, + failure_reason TEXT, + revision INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE jobs ( + job_id TEXT PRIMARY KEY, + capture_id TEXT REFERENCES captures(capture_id) ON DELETE RESTRICT, + kind TEXT NOT NULL CHECK(length(kind) <= 100), + state TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0 CHECK(attempt >= 0 AND attempt <= 20), + max_attempts INTEGER NOT NULL CHECK(max_attempts BETWEEN 1 AND 20), + lease_owner TEXT, lease_expires_at TEXT, last_error_id TEXT, + revision INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX jobs_available ON jobs(state, lease_expires_at); +CREATE TABLE ocr_runs ( + run_id TEXT PRIMARY KEY, capture_id TEXT NOT NULL REFERENCES captures(capture_id), + engine_id TEXT NOT NULL, engine_version TEXT NOT NULL, created_at TEXT NOT NULL, + state TEXT NOT NULL, payload_json TEXT NOT NULL CHECK(length(payload_json) <= 1000000) +); +CREATE TABLE regions ( + region_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES ocr_runs(run_id), + region_json TEXT NOT NULL CHECK(length(region_json) <= 10000), created_at TEXT NOT NULL +); +CREATE TABLE classifications ( + classification_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES ocr_runs(run_id), + classification_json TEXT NOT NULL CHECK(length(classification_json) <= 10000), created_at TEXT NOT NULL +); +CREATE TABLE extraction_candidates ( + candidate_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES ocr_runs(run_id), + candidate_json TEXT NOT NULL CHECK(length(candidate_json) <= 1000000), created_at TEXT NOT NULL +); +CREATE TABLE routing_decisions ( + decision_id TEXT PRIMARY KEY, candidate_id TEXT REFERENCES extraction_candidates(candidate_id), + decision_json TEXT NOT NULL CHECK(length(decision_json) <= 10000), created_at TEXT NOT NULL +); +CREATE TABLE review_events ( + event_id TEXT PRIMARY KEY, candidate_id TEXT REFERENCES extraction_candidates(candidate_id), + event_json TEXT NOT NULL CHECK(length(event_json) <= 10000), created_at TEXT NOT NULL +); +CREATE TABLE lineage ( + lineage_id TEXT PRIMARY KEY, child_id TEXT NOT NULL, parent_id TEXT NOT NULL, + relation TEXT NOT NULL CHECK(length(relation) <= 100), created_at TEXT NOT NULL +); +CREATE TABLE exports ( + export_id TEXT PRIMARY KEY, state TEXT NOT NULL, manifest_json TEXT NOT NULL CHECK(length(manifest_json) <= 1000000), created_at TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE runtime_failures ( + failure_id TEXT PRIMARY KEY, subject_type TEXT NOT NULL, subject_id TEXT NOT NULL, + code TEXT NOT NULL, message TEXT NOT NULL CHECK(length(message) <= 512), created_at TEXT NOT NULL +); diff --git a/tools/game-data-capture/migrations/002_capture_corpus.sql b/tools/game-data-capture/migrations/002_capture_corpus.sql new file mode 100644 index 0000000..a83ce79 --- /dev/null +++ b/tools/game-data-capture/migrations/002_capture_corpus.sql @@ -0,0 +1,38 @@ +-- Durable browser-corpus projections and append-only review evidence. +CREATE TABLE capture_labels ( + capture_id TEXT PRIMARY KEY REFERENCES captures(capture_id) ON DELETE RESTRICT, + classification_json TEXT NOT NULL CHECK(length(classification_json) BETWEEN 2 AND 4096), + revision INTEGER NOT NULL DEFAULT 0 CHECK(revision >= 0), + labeled_at TEXT NOT NULL +); +CREATE TABLE capture_label_evidence ( + evidence_id INTEGER PRIMARY KEY, + capture_id TEXT NOT NULL REFERENCES captures(capture_id) ON DELETE RESTRICT, + replaced_revision INTEGER NOT NULL CHECK(replaced_revision >= 0), + classification_json TEXT NOT NULL CHECK(length(classification_json) BETWEEN 2 AND 4096), + replaced_at TEXT NOT NULL, + UNIQUE(capture_id, replaced_revision) +); +CREATE INDEX capture_label_evidence_capture ON capture_label_evidence(capture_id, evidence_id); +CREATE INDEX captures_blob_sha256 ON captures(blob_sha256); +CREATE INDEX jobs_capture_id ON jobs(capture_id); +CREATE INDEX ocr_runs_capture_id ON ocr_runs(capture_id); +CREATE INDEX runtime_failures_capture_subject ON runtime_failures(subject_type, subject_id); +CREATE INDEX lineage_child_id ON lineage(child_id); +CREATE INDEX lineage_parent_id ON lineage(parent_id); +CREATE TABLE capture_rejections ( + rejection_id INTEGER PRIMARY KEY, + capture_id TEXT NOT NULL UNIQUE REFERENCES captures(capture_id) ON DELETE RESTRICT, + rejected_at TEXT NOT NULL +); +CREATE TABLE capture_deletion_intents ( + blob_sha256 TEXT PRIMARY KEY REFERENCES blobs(sha256) ON DELETE RESTRICT, + capture_id TEXT NOT NULL UNIQUE CHECK(length(capture_id) BETWEEN 8 AND 64), + byte_size INTEGER NOT NULL CHECK(byte_size > 0 AND byte_size <= 67108864), + capture_state TEXT NOT NULL CHECK(length(capture_state) BETWEEN 1 AND 32), + received_at TEXT NOT NULL CHECK(length(received_at) BETWEEN 1 AND 40), + receipt_token TEXT NOT NULL UNIQUE CHECK(length(receipt_token) BETWEEN 1 AND 128), + failure_reason TEXT, + revision INTEGER NOT NULL CHECK(revision >= 0), + created_at TEXT NOT NULL CHECK(length(created_at) BETWEEN 1 AND 40) +); diff --git a/tools/game-data-capture/pyproject.toml b/tools/game-data-capture/pyproject.toml new file mode 100644 index 0000000..781eb7c --- /dev/null +++ b/tools/game-data-capture/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["hatchling==1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "shd-game-data-capture" +version = "0.1.0" +description = "Local, privacy-preserving game-data capture utility contracts." +requires-python = ">=3.12,<3.13" +license = { text = "UNLICENSED" } +authors = [{ name = "SHD.build" }] +dependencies = [ + "fastapi==0.115.12", + "jinja2==3.1.6", + "python-multipart==0.0.20", + "uvicorn==0.34.0", +] + +[project.scripts] +shd-capture = "game_data_capture.cli:main" + +[dependency-groups] +dev = [ + "httpx==0.28.1", + "jsonschema==4.25.1", + "mypy==1.15.0", + "pytest==8.3.5", + "ruff==0.11.0", + "types-jsonschema==4.25.1.20251009", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/game_data_capture"] + +[tool.ruff] +target-version = "py312" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM", "RUF"] + +[tool.mypy] +python_version = "3.12" +strict = true +mypy_path = ["src"] +warn_unreachable = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +addopts = "-ra" diff --git a/tools/game-data-capture/src/game_data_capture/__init__.py b/tools/game-data-capture/src/game_data_capture/__init__.py new file mode 100644 index 0000000..0c1abb3 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/__init__.py @@ -0,0 +1,6 @@ +"""Contracts for the local SHD.build game-data capture utility. + +Importing this package does not inspect the environment or initialize runtime resources. +""" + +__version__ = "0.1.0" diff --git a/tools/game-data-capture/src/game_data_capture/__main__.py b/tools/game-data-capture/src/game_data_capture/__main__.py new file mode 100644 index 0000000..bfd3004 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/__main__.py @@ -0,0 +1,6 @@ +"""Module entry point for ``python -m game_data_capture``.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/game-data-capture/src/game_data_capture/cli.py b/tools/game-data-capture/src/game_data_capture/cli.py new file mode 100644 index 0000000..e4935b5 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/cli.py @@ -0,0 +1,163 @@ +"""CLI contracts for the capture utility.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from dataclasses import replace +from pathlib import Path + +from game_data_capture.config import AppConfig, ConfigError, ServerConfig, load_config +from game_data_capture.doctor import doctor_as_dict, run_doctor +from game_data_capture.paths import PathConfigurationError, RuntimePaths, resolve_runtime_paths +from game_data_capture.runtime import CaptureRuntime, RuntimeFailure + +EXIT_OK = 0 +EXIT_CONFIG_ERROR = 2 +EXIT_UNIMPLEMENTED = 3 +EXIT_RUNTIME_ERROR = 4 + + +def _add_config_argument(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--config", type=Path, help="absolute path to a TOML configuration file") + + +def _port_argument(raw_value: str) -> int: + try: + port = int(raw_value) + except ValueError as error: + raise argparse.ArgumentTypeError("port must be an integer from 1024 to 65535") from error + if not 1024 <= port <= 65535: + raise argparse.ArgumentTypeError("port must be an integer from 1024 to 65535") + return port + + +def build_parser() -> argparse.ArgumentParser: + """Build the stable command surface without initializing runtime resources.""" + + parser = argparse.ArgumentParser( + prog="shd-capture", description="Local SHD.build game-data capture" + ) + subcommands = parser.add_subparsers(dest="command", required=True) + + doctor = subcommands.add_parser("doctor", help="report local prerequisite status") + _add_config_argument(doctor) + doctor.add_argument("--json", action="store_true", help="emit JSON diagnostics") + + serve = subcommands.add_parser("serve", help="start the localhost review service") + _add_config_argument(serve) + serve.add_argument("--host", help="loopback IP literal override") + serve.add_argument("--port", type=_port_argument, help="TCP port override") + + watch = subcommands.add_parser("watch", help="watch the Wayland clipboard for images") + _add_config_argument(watch) + watch_modes = watch.add_mutually_exclusive_group() + watch_modes.add_argument("--arm", action="store_true", help="arm live clipboard capture") + watch_modes.add_argument( + "--dry-run", action="store_true", help="validate the capture runtime plan" + ) + + ingest_image = subcommands.add_parser("ingest-image", help="stage one image for capture") + _add_config_argument(ingest_image) + ingest_image.add_argument("image", type=Path) + + import_video = subcommands.add_parser("import-video", help="stage a video for frame extraction") + _add_config_argument(import_video) + import_video.add_argument("video", type=Path) + + export = subcommands.add_parser("export", help="write a versioned ISAC handoff") + _add_config_argument(export) + export.add_argument("destination", type=Path) + + verify = subcommands.add_parser("verify", help="verify an evidence archive") + _add_config_argument(verify) + verify.add_argument("archive", type=Path) + + evaluate = subcommands.add_parser("eval", help="evaluate local OCR and classification") + _add_config_argument(evaluate) + evaluate.add_argument("corpus", type=Path) + + purge = subcommands.add_parser("purge", help="purge retained local data") + _add_config_argument(purge) + purge.add_argument("--confirm", action="store_true", help="confirm a future destructive purge") + + return parser + + +def _load_cli_config(config_path: Path | None) -> tuple[AppConfig, RuntimePaths]: + paths = resolve_runtime_paths() + return load_config(config_path=config_path, paths=paths), paths + + +def _doctor(arguments: argparse.Namespace) -> int: + config, paths = _load_cli_config(arguments.config) + diagnostics = doctor_as_dict(run_doctor(config, paths)) + if arguments.json: + print(json.dumps(diagnostics, separators=(",", ":"), sort_keys=True)) + else: + for diagnostic in diagnostics: + print(f"{diagnostic['name']}: {diagnostic['status']} ({diagnostic['detail']})") + return EXIT_OK + + +def _serve_contract(arguments: argparse.Namespace) -> int: + config, paths = _load_cli_config(arguments.config) + host = arguments.host if arguments.host is not None else config.server.host + port = arguments.port if arguments.port is not None else config.server.port + configured = replace(config, server=ServerConfig(host=host, port=port)) + CaptureRuntime(configured, paths).serve() + return EXIT_OK + + +def _unimplemented(command: str) -> int: + print( + f"{command}: subsystem is not implemented; requires gdc-runtime-integration", + file=sys.stderr, + ) + return EXIT_UNIMPLEMENTED + + +def _watch(arguments: argparse.Namespace) -> int: + config, paths = _load_cli_config(arguments.config) + if not arguments.arm and not arguments.dry_run: + return _unimplemented("watch") + runtime = CaptureRuntime(config, paths) + if arguments.dry_run: + runtime.validate_plan() + print("watch dry-run: capture runtime plan is valid") + return EXIT_OK + runtime.watch() + return EXIT_OK + + +def _ingest_image(arguments: argparse.Namespace) -> int: + config, paths = _load_cli_config(arguments.config) + CaptureRuntime(config, paths).ingest_image(arguments.image) + print("image capture stored") + return EXIT_OK + + +def main(argv: Sequence[str] | None = None) -> int: + """Run one CLI command and return a stable process exit status.""" + + parser = build_parser() + arguments = parser.parse_args(argv) + try: + if arguments.command == "doctor": + return _doctor(arguments) + if arguments.command == "serve": + return _serve_contract(arguments) + if arguments.command == "watch": + return _watch(arguments) + if arguments.command == "ingest-image": + return _ingest_image(arguments) + _load_cli_config(arguments.config) + return _unimplemented(arguments.command) + except (ConfigError, PathConfigurationError) as error: + print(f"configuration error: {error}", file=sys.stderr) + return EXIT_CONFIG_ERROR + except RuntimeFailure: + print("capture runtime failed", file=sys.stderr) + return EXIT_RUNTIME_ERROR diff --git a/tools/game-data-capture/src/game_data_capture/config.py b/tools/game-data-capture/src/game_data_capture/config.py new file mode 100644 index 0000000..bf648dd --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/config.py @@ -0,0 +1,216 @@ +"""Typed, bounded configuration for the capture runtime. + +Configuration parsing reads an explicit TOML file only when a caller invokes +``load_config``. Importing this module has no filesystem or environment effects. +""" + +from __future__ import annotations + +import ipaddress +import os +import re +import tomllib +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from game_data_capture.paths import RuntimePaths, resolve_runtime_paths + +_ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]{0,63}$") + + +class ConfigError(ValueError): + """A bounded, safe-to-display configuration error.""" + + +def _error(field_name: str, constraint: str) -> ConfigError: + return ConfigError(f"Invalid configuration: {field_name} {constraint}.") + + +def _bounded_int(value: object, field_name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: + raise _error(field_name, f"must be an integer from {minimum} to {maximum}") + return value + + +def _mapping(value: object, field_name: str) -> Mapping[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise _error(field_name, "must be a table") + return value + + +def _only_keys(section: Mapping[str, object], section_name: str, allowed: set[str]) -> None: + if set(section) - allowed: + raise _error(section_name, "contains an unsupported setting") + + +def _string(value: object, field_name: str, maximum: int) -> str: + if not isinstance(value, str) or not value or len(value) > maximum: + raise _error(field_name, f"must be a non-empty string up to {maximum} characters") + return value + + +@dataclass(frozen=True, slots=True) +class ServerConfig: + host: str = "127.0.0.1" + port: int = 8765 + + def __post_init__(self) -> None: + try: + address = ipaddress.ip_address(self.host) + except ValueError as error: + raise _error("server.host", "must be a loopback IP literal") from error + if not address.is_loopback: + raise _error("server.host", "must be a loopback IP literal") + _bounded_int(self.port, "server.port", 1024, 65535) + + +@dataclass(frozen=True, slots=True) +class LifecycleConfig: + """One-instance settings consumed later by the runtime integration layer.""" + + instance_name: str = "shd-capture" + lock_timeout_seconds: int = 5 + shutdown_timeout_seconds: int = 30 + + def __post_init__(self) -> None: + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,31}", self.instance_name): + raise _error( + "lifecycle.instance_name", "must use lowercase letters, digits, and hyphens" + ) + _bounded_int(self.lock_timeout_seconds, "lifecycle.lock_timeout_seconds", 0, 60) + _bounded_int(self.shutdown_timeout_seconds, "lifecycle.shutdown_timeout_seconds", 1, 300) + + +@dataclass(frozen=True, slots=True) +class CaptureLimits: + max_image_bytes: int = 32 * 1024 * 1024 + max_video_bytes: int = 4 * 1024 * 1024 * 1024 + max_queue_items: int = 64 + max_ocr_text_characters: int = 20_000 + + def __post_init__(self) -> None: + _bounded_int(self.max_image_bytes, "limits.max_image_bytes", 1, 64 * 1024 * 1024) + _bounded_int(self.max_video_bytes, "limits.max_video_bytes", 1, 16 * 1024 * 1024 * 1024) + _bounded_int(self.max_queue_items, "limits.max_queue_items", 1, 256) + _bounded_int(self.max_ocr_text_characters, "limits.max_ocr_text_characters", 1, 100_000) + + +@dataclass(frozen=True, slots=True) +class OpenAIConfig: + api_key_env: str = "OPENAI_API_KEY" + model: str = "gpt-4.1-mini" + + def __post_init__(self) -> None: + if not _ENVIRONMENT_NAME.fullmatch(self.api_key_env): + raise _error("openai.api_key_env", "must be an environment variable name") + _string(self.model, "openai.model", 128) + + def is_configured(self, environment: Mapping[str, str] | None = None) -> bool: + env = os.environ if environment is None else environment + return bool(env.get(self.api_key_env)) + + +@dataclass(frozen=True, slots=True) +class AppConfig: + server: ServerConfig = field(default_factory=ServerConfig) + lifecycle: LifecycleConfig = field(default_factory=LifecycleConfig) + limits: CaptureLimits = field(default_factory=CaptureLimits) + openai: OpenAIConfig = field(default_factory=OpenAIConfig) + + +def _section(document: Mapping[str, object], name: str, allowed: set[str]) -> Mapping[str, object]: + value = document.get(name, {}) + section = _mapping(value, name) + _only_keys(section, name, allowed) + return section + + +def parse_config(document: Mapping[str, object]) -> AppConfig: + """Parse a TOML-shaped mapping with an allowlist of configuration keys.""" + + _only_keys(document, "root", {"server", "lifecycle", "limits", "openai"}) + server = _section(document, "server", {"host", "port"}) + lifecycle = _section( + document, "lifecycle", {"instance_name", "lock_timeout_seconds", "shutdown_timeout_seconds"} + ) + limits = _section( + document, + "limits", + {"max_image_bytes", "max_video_bytes", "max_queue_items", "max_ocr_text_characters"}, + ) + openai = _section(document, "openai", {"api_key_env", "model"}) + + return AppConfig( + server=ServerConfig( + host=_string(server.get("host", "127.0.0.1"), "server.host", 64), + port=_bounded_int(server.get("port", 8765), "server.port", 1024, 65535), + ), + lifecycle=LifecycleConfig( + instance_name=_string( + lifecycle.get("instance_name", "shd-capture"), "lifecycle.instance_name", 32 + ), + lock_timeout_seconds=_bounded_int( + lifecycle.get("lock_timeout_seconds", 5), "lifecycle.lock_timeout_seconds", 0, 60 + ), + shutdown_timeout_seconds=_bounded_int( + lifecycle.get("shutdown_timeout_seconds", 30), + "lifecycle.shutdown_timeout_seconds", + 1, + 300, + ), + ), + limits=CaptureLimits( + max_image_bytes=_bounded_int( + limits.get("max_image_bytes", 32 * 1024 * 1024), + "limits.max_image_bytes", + 1, + 64 * 1024 * 1024, + ), + max_video_bytes=_bounded_int( + limits.get("max_video_bytes", 4 * 1024 * 1024 * 1024), + "limits.max_video_bytes", + 1, + 16 * 1024 * 1024 * 1024, + ), + max_queue_items=_bounded_int( + limits.get("max_queue_items", 64), "limits.max_queue_items", 1, 256 + ), + max_ocr_text_characters=_bounded_int( + limits.get("max_ocr_text_characters", 20_000), + "limits.max_ocr_text_characters", + 1, + 100_000, + ), + ), + openai=OpenAIConfig( + api_key_env=_string( + openai.get("api_key_env", "OPENAI_API_KEY"), "openai.api_key_env", 64 + ), + model=_string(openai.get("model", "gpt-4.1-mini"), "openai.model", 128), + ), + ) + + +def load_config(config_path: Path | None = None, paths: RuntimePaths | None = None) -> AppConfig: + """Read optional TOML configuration from the XDG config location. + + An absent file selects safe defaults. Parse failures intentionally omit source values and paths. + """ + + selected_path = ( + config_path if config_path is not None else (paths or resolve_runtime_paths()).config_file + ) + if not selected_path.is_absolute() or len(str(selected_path)) > 240: + raise _error("config path", "must be an absolute path within 240 characters") + if not selected_path.exists(): + return AppConfig() + try: + with selected_path.open("rb") as handle: + document: Any = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as error: + raise _error("config file", "could not be read") from error + if not isinstance(document, dict): + raise _error("root", "must be a table") + return parse_config(document) diff --git a/tools/game-data-capture/src/game_data_capture/doctor.py b/tools/game-data-capture/src/game_data_capture/doctor.py new file mode 100644 index 0000000..8e4b6a3 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/doctor.py @@ -0,0 +1,84 @@ +"""Side-effect-limited local diagnostics for capture prerequisites.""" + +from __future__ import annotations + +import importlib.util +import shutil +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from os import environ + +from game_data_capture.config import AppConfig +from game_data_capture.paths import ( + PathConfigurationError, + RuntimePaths, + prepare_private_directories, +) + + +@dataclass(frozen=True, slots=True) +class Diagnostic: + name: str + status: str + detail: str + + +def _executable_diagnostic(name: str) -> Diagnostic: + return Diagnostic( + name=name, + status="available" if shutil.which(name) else "unavailable", + detail="executable found on PATH" if shutil.which(name) else "executable not found on PATH", + ) + + +def _gpu_diagnostic(environment: Mapping[str, str]) -> Diagnostic: + configured_provider = environment.get("GDC_OCR_PROVIDER") + if configured_provider: + return Diagnostic( + "gpu_provider", "unknown", "provider configured; runtime probe is deferred" + ) + if importlib.util.find_spec("torch") is not None: + return Diagnostic( + "gpu_provider", "unknown", "torch is installed; runtime probe is deferred" + ) + return Diagnostic("gpu_provider", "unknown", "no provider selected; runtime probe is deferred") + + +def _storage_diagnostic(paths: RuntimePaths) -> Diagnostic: + try: + prepare_private_directories(paths) + except (OSError, PathConfigurationError): + return Diagnostic( + "storage_permissions", "unavailable", "private XDG directories are not usable" + ) + return Diagnostic( + "storage_permissions", "available", "private XDG directories use owner-only access" + ) + + +def run_doctor( + config: AppConfig, paths: RuntimePaths, environment: Mapping[str, str] | None = None +) -> tuple[Diagnostic, ...]: + """Inspect local prerequisites without launching processes or making network calls.""" + + env = environ if environment is None else environment + return ( + _executable_diagnostic("niri"), + _executable_diagnostic("wl-paste"), + _executable_diagnostic("ffmpeg"), + _gpu_diagnostic(env), + _storage_diagnostic(paths), + Diagnostic( + "openai", + "configured" if config.openai.is_configured(env) else "missing", + "environment variable is present" + if config.openai.is_configured(env) + else "environment variable is absent", + ), + ) + + +def doctor_as_dict(diagnostics: tuple[Diagnostic, ...]) -> list[dict[str, str]]: + """Return diagnostics in a serialization-ready form without paths or secret values.""" + + return [asdict(diagnostic) for diagnostic in diagnostics] diff --git a/tools/game-data-capture/src/game_data_capture/domain/__init__.py b/tools/game-data-capture/src/game_data_capture/domain/__init__.py new file mode 100644 index 0000000..c586cf4 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/__init__.py @@ -0,0 +1,43 @@ +"""Side-effect-free domain contracts for the game data capture utility.""" + +from .state_machines import InvalidStateTransition, StateMachine, require_transition +from .taxonomy import ( + ABSTENTION_REASONS, + GEAR_SLOTS, + RECORD_TYPES_BY_VIEW, + SCREEN_VIEWS, + AbstentionReason, + Family, + GearSlot, + RecordType, +) +from .validation import ( + ContractValidationError, + ensure_supported_version, + validate_archive_document, + validate_archive_references, + validate_classification, + validate_handoff_document, + validate_handoff_references, +) + +__all__ = [ + "ABSTENTION_REASONS", + "GEAR_SLOTS", + "RECORD_TYPES_BY_VIEW", + "SCREEN_VIEWS", + "AbstentionReason", + "ContractValidationError", + "Family", + "GearSlot", + "InvalidStateTransition", + "RecordType", + "StateMachine", + "ensure_supported_version", + "require_transition", + "validate_archive_document", + "validate_archive_references", + "validate_classification", + "validate_handoff_document", + "validate_handoff_references", +] diff --git a/tools/game-data-capture/src/game_data_capture/domain/models.py b/tools/game-data-capture/src/game_data_capture/domain/models.py new file mode 100644 index 0000000..a9ac71a --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/models.py @@ -0,0 +1,118 @@ +"""Typed Python views of the portable contract structures.""" + +import typing + +type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] +type MissingReason = typing.Literal[ + "not_visible", + "occluded", + "unreadable", + "not_present_on_screen", + "unsupported", + "ambiguous", + "extraction_failed", + "not_applicable", + "redacted", +] + + +class SourceEngine(typing.TypedDict): + kind: typing.Literal["local_ocr", "openai_vision", "classifier", "manual", "video_decoder"] + engine_id: str + engine_version: str + + +class NormalizedRegion(typing.TypedDict): + coordinate_space: typing.Literal["normalized"] + x: float + y: float + width: float + height: float + + +class PixelRegion(typing.TypedDict): + coordinate_space: typing.Literal["pixels"] + x: int + y: int + width: int + height: int + + +type BoundingRegion = NormalizedRegion | PixelRegion + + +class ValueState(typing.TypedDict): + raw_text: str | None + normalized_value: JsonValue + missing_reason: MissingReason | None + + +class Correction(typing.TypedDict): + correction_id: str + corrected_at: str + actor: typing.Literal["local_user", "system_reprocess"] + reason: str + before: ValueState + after: ValueState + + +class EvidenceValue(ValueState): + evidence_id: str + field_path: str + confidence: float + source_engine: SourceEngine + image_artifact_id: str + image_sha256: str + region: BoundingRegion + correction_history: list[Correction] + + +class FieldClaim(typing.TypedDict): + path: str + value: JsonValue + missing_reason: MissingReason | None + evidence_refs: list[str] + uncertainty_note: str | None + + +class ScreenshotClassification(typing.TypedDict): + family: typing.Literal[ + "weapon", "gear", "skill_chip", "os_protocol", "specialization", "unknown" + ] + view: str + gear_slot: ( + typing.Literal["Mask", "Gloves", "Backpack", "Body Armor", "Holster", "Knee Pads"] | None + ) + record_types: list[str] + confidence: float + abstention_reason: str | None + + +class ObservedContext(typing.TypedDict): + level: FieldClaim + quality: FieldClaim + tier: FieldClaim + upgrade_state: FieldClaim + + +class ObservedItemInstance(typing.TypedDict): + """Values shown for one configured item; they are not reusable definitions.""" + + instance_id: str + family: str + source_screen_ids: list[str] + observed_context: ObservedContext + fields: list[FieldClaim] + + +class CanonicalDefinitionCandidate(typing.TypedDict): + """Evidence-backed reusable claims, kept separate from the observed item.""" + + candidate_id: str + record_type: str + source_instance_ids: list[str] + evidence_refs: list[str] + fields: list[FieldClaim] + completeness: typing.Literal["complete", "incomplete", "unsupported"] + relationship_status: typing.Literal["evidenced", "unresolved", "unsupported", "not_applicable"] + mechanic_status: typing.Literal["evidenced", "unresolved", "unsupported", "not_applicable"] diff --git a/tools/game-data-capture/src/game_data_capture/domain/ports.py b/tools/game-data-capture/src/game_data_capture/domain/ports.py new file mode 100644 index 0000000..4cfc6c6 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/ports.py @@ -0,0 +1,214 @@ +"""Trust-boundary ports. Implementations belong to downstream tasks.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from math import isfinite +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class ResourceBudget: + deadline_monotonic: float | None + max_input_bytes: int + max_output_bytes: int + max_items: int + + def __post_init__(self) -> None: + limits: tuple[tuple[str, object], ...] = ( + ("max_input_bytes", self.max_input_bytes), + ("max_output_bytes", self.max_output_bytes), + ("max_items", self.max_items), + ) + for name, value in limits: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + deadline: object = self.deadline_monotonic + if deadline is not None: + if isinstance(deadline, bool) or not isinstance(deadline, int | float): + raise ValueError("deadline_monotonic must be a finite number or null") + if not isfinite(deadline): + raise ValueError("deadline_monotonic must be finite or null") + + +class CancellationToken(Protocol): + def is_cancelled(self) -> bool: ... + + def raise_if_cancelled(self) -> None: ... + + +class Clock(Protocol): + def now_utc(self) -> datetime: ... + + def monotonic(self) -> float: ... + + +class IdGenerator(Protocol): + def new_id(self) -> str: ... + + +@dataclass(frozen=True, slots=True) +class BlobReceipt: + sha256: str + byte_size: int + media_type: str + + +class BlobStorage(Protocol): + """Stores immutable bytes by digest without exposing filesystem paths.""" + + def put( + self, + chunks: Iterable[bytes], + *, + expected_sha256: str | None, + media_type: str, + budget: ResourceBudget, + cancellation: CancellationToken, + ) -> BlobReceipt: ... + + def read( + self, + sha256: str, + *, + budget: ResourceBudget, + cancellation: CancellationToken, + ) -> Iterable[bytes]: ... + + def exists(self, sha256: str) -> bool: ... + + +class Repository[RecordT](Protocol): + def get(self, record_id: str) -> RecordT | None: ... + + def add(self, record: RecordT) -> None: ... + + def replace(self, record: RecordT, *, expected_revision: int) -> None: ... + + +type CaptureRepository = Repository[object] +type JobRepository = Repository[object] +type ExtractionRepository = Repository[object] +type GroupingRepository = Repository[object] +type ReviewRepository = Repository[object] +type ExportRepository = Repository[object] +type ErrorRepository = Repository[object] + + +class UnitOfWork(Protocol): + captures: CaptureRepository + jobs: JobRepository + extractions: ExtractionRepository + groupings: GroupingRepository + reviews: ReviewRepository + exports: ExportRepository + errors: ErrorRepository + + def commit(self) -> None: ... + + def rollback(self) -> None: ... + + +@dataclass(frozen=True, slots=True) +class ImageInput: + artifact_id: str + sha256: str + media_type: str + byte_size: int + + +@dataclass(frozen=True, slots=True) +class EngineResult: + engine_id: str + engine_version: str + payload: Mapping[str, object] + + +class OcrEngine(Protocol): + def extract( + self, + image: ImageInput, + *, + regions: Sequence[Mapping[str, int | float | str]], + budget: ResourceBudget, + cancellation: CancellationToken, + ) -> EngineResult: ... + + +class VisionFallback(Protocol): + """Accepts bounded artifacts and never receives credentials in a request.""" + + def extract( + self, + images: Sequence[ImageInput], + *, + response_schema: Mapping[str, object], + request_fingerprint: str, + budget: ResourceBudget, + cancellation: CancellationToken, + ) -> EngineResult: ... + + +@dataclass(frozen=True, slots=True) +class SubprocessRequest: + tool: SubprocessTool + arguments: tuple[str, ...] + environment: Mapping[SubprocessEnvironmentKey, str] + stdin: bytes | None + budget: ResourceBudget + + +class SubprocessTool(StrEnum): + WL_PASTE = "wl-paste" + FFMPEG = "ffmpeg" + FFPROBE = "ffprobe" + + +class SubprocessEnvironmentKey(StrEnum): + WAYLAND_DISPLAY = "WAYLAND_DISPLAY" + XDG_RUNTIME_DIR = "XDG_RUNTIME_DIR" + PATH = "PATH" + HOME = "HOME" + LANG = "LANG" + LC_ALL = "LC_ALL" + + +@dataclass(frozen=True, slots=True) +class SubprocessResult: + return_code: int + stdout: bytes + stderr: bytes + timed_out: bool + cancelled: bool + + +class SubprocessRunner(Protocol): + """Runs an argument array without a shell and with an explicit environment.""" + + def run( + self, + request: SubprocessRequest, + *, + cancellation: CancellationToken, + ) -> SubprocessResult: ... + + +@dataclass(frozen=True, slots=True) +class FrameScore: + timestamp_ms: int + stability: float + sharpness: float + ui_coverage: float + + +class FrameScorer(Protocol): + def score( + self, + frames: Sequence[ImageInput], + *, + budget: ResourceBudget, + cancellation: CancellationToken, + ) -> Sequence[FrameScore]: ... diff --git a/tools/game-data-capture/src/game_data_capture/domain/state_machines.py b/tools/game-data-capture/src/game_data_capture/domain/state_machines.py new file mode 100644 index 0000000..92346c6 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/state_machines.py @@ -0,0 +1,211 @@ +"""State vocabularies and allowed transitions for recoverable workflows.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from types import MappingProxyType +from typing import Final + + +class CaptureState(StrEnum): + RECEIVED = "received" + SPOOLING = "spooling" + STORED = "stored" + DEDUPLICATED = "deduplicated" + QUEUED = "queued" + FAILED = "failed" + CANCELLED = "cancelled" + + +class JobState(StrEnum): + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class ExtractionState(StrEnum): + PENDING = "pending" + LOCAL_PROCESSING = "local_processing" + FALLBACK_PENDING = "fallback_pending" + FALLBACK_PROCESSING = "fallback_processing" + CANDIDATE_READY = "candidate_ready" + ABSTAINED = "abstained" + FAILED = "failed" + CANCELLED = "cancelled" + + +class GroupingState(StrEnum): + UNGROUPED = "ungrouped" + PROPOSED = "proposed" + GROUPED = "grouped" + SPLIT = "split" + SUPERSEDED = "superseded" + REJECTED = "rejected" + CANCELLED = "cancelled" + + +class ReviewState(StrEnum): + PENDING = "pending" + IN_REVIEW = "in_review" + CHANGES_REQUESTED = "changes_requested" + APPROVED = "approved" + REJECTED = "rejected" + CANCELLED = "cancelled" + + +class ExportState(StrEnum): + NOT_REQUESTED = "not_requested" + QUEUED = "queued" + BUILDING = "building" + READY = "ready" + FAILED = "failed" + CANCELLED = "cancelled" + EXPIRED = "expired" + + +class ErrorState(StrEnum): + OPEN = "open" + RETRYABLE = "retryable" + RETRYING = "retrying" + RECOVERED = "recovered" + TERMINAL = "terminal" + DISMISSED = "dismissed" + + +@dataclass(frozen=True, slots=True) +class StateMachine: + name: str + transitions: Mapping[str, frozenset[str]] + + def allows(self, current: str, target: str) -> bool: + return target in self.transitions.get(current, frozenset()) + + +def _machine(name: str, transitions: dict[str, set[str]]) -> StateMachine: + return StateMachine( + name=name, + transitions=MappingProxyType( + {state: frozenset(targets) for state, targets in transitions.items()} + ), + ) + + +STATE_MACHINES: Final[Mapping[str, StateMachine]] = MappingProxyType( + { + "capture": _machine( + "capture", + { + "received": {"spooling", "deduplicated", "failed", "cancelled"}, + "spooling": {"stored", "failed", "cancelled"}, + "stored": {"queued", "failed"}, + "deduplicated": set(), + "queued": set(), + "failed": {"spooling", "cancelled"}, + "cancelled": set(), + }, + ), + "job": _machine( + "job", + { + "queued": {"running", "cancelled"}, + "running": {"succeeded", "retry_wait", "failed", "cancelled"}, + "retry_wait": {"queued", "failed", "cancelled"}, + "succeeded": set(), + "failed": {"queued"}, + "cancelled": set(), + }, + ), + "extraction": _machine( + "extraction", + { + "pending": {"local_processing", "cancelled"}, + "local_processing": { + "candidate_ready", + "fallback_pending", + "abstained", + "failed", + "cancelled", + }, + "fallback_pending": {"fallback_processing", "cancelled"}, + "fallback_processing": { + "candidate_ready", + "abstained", + "failed", + "cancelled", + }, + "candidate_ready": set(), + "abstained": set(), + "failed": {"pending"}, + "cancelled": set(), + }, + ), + "grouping": _machine( + "grouping", + { + "ungrouped": {"proposed", "cancelled"}, + "proposed": {"grouped", "rejected", "cancelled"}, + "grouped": {"split", "superseded"}, + "split": {"superseded"}, + "superseded": set(), + "rejected": {"ungrouped"}, + "cancelled": set(), + }, + ), + "review": _machine( + "review", + { + "pending": {"in_review", "cancelled"}, + "in_review": { + "changes_requested", + "approved", + "rejected", + "cancelled", + }, + "changes_requested": {"in_review", "rejected", "cancelled"}, + "approved": {"in_review"}, + "rejected": {"in_review"}, + "cancelled": set(), + }, + ), + "export": _machine( + "export", + { + "not_requested": {"queued"}, + "queued": {"building", "cancelled"}, + "building": {"ready", "failed", "cancelled"}, + "ready": {"expired"}, + "failed": {"queued", "cancelled"}, + "cancelled": set(), + "expired": {"queued"}, + }, + ), + "error": _machine( + "error", + { + "open": {"retryable", "terminal", "dismissed"}, + "retryable": {"retrying", "terminal", "dismissed"}, + "retrying": {"recovered", "retryable", "terminal"}, + "recovered": set(), + "terminal": {"dismissed"}, + "dismissed": set(), + }, + ), + } +) + + +class InvalidStateTransition(ValueError): + """Raised when a caller attempts a transition absent from the contract.""" + + +def require_transition(machine_name: str, current: str, target: str) -> None: + machine = STATE_MACHINES.get(machine_name) + if machine is None: + raise InvalidStateTransition(f"Unknown state machine: {machine_name}") + if current not in machine.transitions: + raise InvalidStateTransition(f"Unknown {machine_name} state: {current}") + if not machine.allows(current, target): + raise InvalidStateTransition(f"Invalid {machine_name} transition: {current} -> {target}") diff --git a/tools/game-data-capture/src/game_data_capture/domain/taxonomy.py b/tools/game-data-capture/src/game_data_capture/domain/taxonomy.py new file mode 100644 index 0000000..a56c743 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/taxonomy.py @@ -0,0 +1,122 @@ +"""Frozen screenshot taxonomy and canonical record vocabulary for contract v1.""" + +from collections.abc import Mapping +from enum import StrEnum +from types import MappingProxyType +from typing import Final + + +class Family(StrEnum): + WEAPON = "weapon" + GEAR = "gear" + SKILL_CHIP = "skill_chip" + OS_PROTOCOL = "os_protocol" + SPECIALIZATION = "specialization" + UNKNOWN = "unknown" + + +class GearSlot(StrEnum): + MASK = "Mask" + GLOVES = "Gloves" + BACKPACK = "Backpack" + BODY_ARMOR = "Body Armor" + HOLSTER = "Holster" + KNEE_PADS = "Knee Pads" + + +class AbstentionReason(StrEnum): + LOW_CONFIDENCE = "low_confidence" + AMBIGUOUS_FAMILY = "ambiguous_family" + AMBIGUOUS_VIEW = "ambiguous_view" + UNSUPPORTED_FAMILY = "unsupported_family" + UNSUPPORTED_VIEW = "unsupported_view" + INSUFFICIENT_VISUAL_EVIDENCE = "insufficient_visual_evidence" + OCCLUDED = "occluded" + CORRUPT_IMAGE = "corrupt_image" + RESOURCE_LIMIT = "resource_limit" + CANCELLED = "cancelled" + + +class RecordType(StrEnum): + WEAPON = "weapon" + WEAPON_TALENT = "weapon_talent" + WEAPON_AUGMENTATION_PROFILE = "weapon_augmentation_profile" + GEAR = "gear" + BRAND_SET = "brand_set" + GEAR_TALENT = "gear_talent" + GEAR_AUGMENTATION_PROFILE = "gear_augmentation_profile" + SKILL_CHIP_SET = "skill_chip_set" + SKILL_CHIP_SET_BONUS = "skill_chip_set_bonus" + SKILL_CHIP_ROLL_POOL = "skill_chip_roll_pool" + SKILL_CHIP_TALENT = "skill_chip_talent" + OS_PROTOCOL = "os_protocol" + OS_PROTOCOL_SECONDARY_PROFILE = "os_protocol_secondary_profile" + OS_PROTOCOL_TALENT = "os_protocol_talent" + SPECIALIZATION = "specialization" + SPECIALIZATION_FOCUS = "specialization_focus" + SPECIALIZATION_ABILITY = "specialization_ability" + SPECIALIZATION_TREE_NODE = "specialization_tree_node" + SPECIALIZATION_TREE_RANK = "specialization_tree_rank" + + +SCREEN_VIEWS: Final[Mapping[Family, tuple[str, ...]]] = MappingProxyType( + { + Family.WEAPON: ( + "detail", + "blueprint", + "crafting", + "stat_range", + "talent_grid", + "augmentation", + ), + Family.GEAR: ("detail", "blueprint", "talent", "augmentation"), + Family.SKILL_CHIP: ("detail", "set_bonus", "talent"), + Family.OS_PROTOCOL: ("detail", "secondary_profile", "talent"), + Family.SPECIALIZATION: ( + "overview", + "details", + "ability", + "focus_details", + "focus_tree", + "rank_details", + ), + Family.UNKNOWN: ("unknown",), + } +) + +GEAR_SLOTS: Final[tuple[str, ...]] = tuple(slot.value for slot in GearSlot) +ABSTENTION_REASONS: Final[tuple[str, ...]] = tuple(reason.value for reason in AbstentionReason) + +RECORD_TYPES_BY_VIEW: Final[Mapping[tuple[Family, str], tuple[RecordType, ...]]] = MappingProxyType( + { + (Family.WEAPON, "detail"): (RecordType.WEAPON, RecordType.WEAPON_TALENT), + (Family.WEAPON, "blueprint"): (RecordType.WEAPON, RecordType.WEAPON_TALENT), + (Family.WEAPON, "crafting"): (RecordType.WEAPON,), + (Family.WEAPON, "stat_range"): (RecordType.WEAPON,), + (Family.WEAPON, "talent_grid"): (RecordType.WEAPON_TALENT,), + (Family.WEAPON, "augmentation"): (RecordType.WEAPON_AUGMENTATION_PROFILE,), + (Family.GEAR, "detail"): (RecordType.GEAR, RecordType.BRAND_SET), + (Family.GEAR, "blueprint"): (RecordType.GEAR, RecordType.BRAND_SET), + (Family.GEAR, "talent"): (RecordType.GEAR_TALENT,), + (Family.GEAR, "augmentation"): (RecordType.GEAR_AUGMENTATION_PROFILE,), + (Family.SKILL_CHIP, "detail"): ( + RecordType.SKILL_CHIP_SET, + RecordType.SKILL_CHIP_ROLL_POOL, + ), + (Family.SKILL_CHIP, "set_bonus"): (RecordType.SKILL_CHIP_SET_BONUS,), + (Family.SKILL_CHIP, "talent"): (RecordType.SKILL_CHIP_TALENT,), + (Family.OS_PROTOCOL, "detail"): (RecordType.OS_PROTOCOL,), + (Family.OS_PROTOCOL, "secondary_profile"): (RecordType.OS_PROTOCOL_SECONDARY_PROFILE,), + (Family.OS_PROTOCOL, "talent"): (RecordType.OS_PROTOCOL_TALENT,), + (Family.SPECIALIZATION, "overview"): (RecordType.SPECIALIZATION,), + (Family.SPECIALIZATION, "details"): (RecordType.SPECIALIZATION,), + (Family.SPECIALIZATION, "ability"): (RecordType.SPECIALIZATION_ABILITY,), + (Family.SPECIALIZATION, "focus_details"): (RecordType.SPECIALIZATION_FOCUS,), + (Family.SPECIALIZATION, "focus_tree"): ( + RecordType.SPECIALIZATION_FOCUS, + RecordType.SPECIALIZATION_TREE_NODE, + ), + (Family.SPECIALIZATION, "rank_details"): (RecordType.SPECIALIZATION_TREE_RANK,), + (Family.UNKNOWN, "unknown"): (), + } +) diff --git a/tools/game-data-capture/src/game_data_capture/domain/validation.py b/tools/game-data-capture/src/game_data_capture/domain/validation.py new file mode 100644 index 0000000..9c7c629 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/domain/validation.py @@ -0,0 +1,384 @@ +"""Semantic checks that portable JSON Schema cannot express.""" + +from collections.abc import Iterable, Mapping +from datetime import datetime + +from . import taxonomy + +SUPPORTED_ARCHIVE_SCHEMA_VERSION = 1 +SUPPORTED_HANDOFF_SCHEMA_VERSION = 1 + + +class ContractValidationError(ValueError): + """Raised when a document violates a version or referential contract.""" + + +def ensure_supported_version(document: Mapping[str, object], expected: int) -> None: + version = document.get("schema_version") + if type(version) is not int or version != expected: + raise ContractValidationError( + f"Unsupported schema_version {version!r}; expected {expected}" + ) + + +def _objects(value: object, field: str) -> Iterable[Mapping[str, object]]: + if not isinstance(value, list): + raise ContractValidationError(f"{field} must be an array") + for index, item in enumerate(value): + if not isinstance(item, Mapping): + raise ContractValidationError(f"{field}[{index}] must be an object") + yield item + + +def _ids(items: Iterable[Mapping[str, object]], field: str) -> set[str]: + result: set[str] = set() + for item in items: + value = item.get(field) + if not isinstance(value, str): + raise ContractValidationError(f"{field} must be a string") + if value in result: + raise ContractValidationError(f"Duplicate {field}: {value}") + result.add(value) + return result + + +def _references(item: Mapping[str, object], field: str) -> set[str]: + raw = item.get(field, []) + if not isinstance(raw, list) or any(not isinstance(value, str) for value in raw): + raise ContractValidationError(f"{field} must contain string IDs") + return set(raw) + + +def _require_subset(values: set[str], allowed: set[str], context: str) -> None: + dangling = values - allowed + if dangling: + raise ContractValidationError(f"{context} contains dangling references: {sorted(dangling)}") + + +def validate_classification(classification: Mapping[str, object]) -> None: + family_raw = classification.get("family") + if not isinstance(family_raw, str): + raise ContractValidationError("Unknown classification family") + try: + family = taxonomy.Family(family_raw) + except ValueError as error: + raise ContractValidationError("Unknown classification family") from error + view = classification.get("view") + if not isinstance(view, str) or view not in taxonomy.SCREEN_VIEWS[family]: + raise ContractValidationError(f"Unknown {family.value} view: {view!r}") + confidence = classification.get("confidence") + if ( + isinstance(confidence, bool) + or not isinstance(confidence, int | float) + or not 0 <= confidence <= 1 + ): + raise ContractValidationError("Classification confidence must be between 0 and 1") + gear_slot = classification.get("gear_slot") + if family is taxonomy.Family.GEAR: + if gear_slot not in taxonomy.GEAR_SLOTS: + raise ContractValidationError(f"Unknown Gear slot: {gear_slot!r}") + elif gear_slot is not None: + raise ContractValidationError("Only Gear classifications may set gear_slot") + record_types = classification.get("record_types") + if not isinstance(record_types, list) or any( + not isinstance(record_type, str) for record_type in record_types + ): + raise ContractValidationError("record_types must contain strings") + if len(record_types) != len(set(record_types)): + raise ContractValidationError("record_types must be unique") + allowed = {record_type.value for record_type in taxonomy.RECORD_TYPES_BY_VIEW[(family, view)]} + if family is taxonomy.Family.UNKNOWN: + if record_types: + raise ContractValidationError("Unknown screens cannot claim canonical record types") + reason = classification.get("abstention_reason") + if reason not in taxonomy.ABSTENTION_REASONS: + raise ContractValidationError("Unknown screens require an abstention reason") + else: + if not record_types or not set(record_types).issubset(allowed): + raise ContractValidationError( + f"record_types are incompatible with {family.value}/{view}" + ) + if classification.get("abstention_reason") is not None: + raise ContractValidationError("Known screens cannot set abstention_reason") + + +def _validate_evidence(value: Mapping[str, object]) -> None: + _validate_value_state(value, "evidence") + confidence = value.get("confidence") + if ( + isinstance(confidence, bool) + or not isinstance(confidence, int | float) + or not 0 <= confidence <= 1 + ): + raise ContractValidationError("Evidence confidence must be between 0 and 1") + region = value.get("region") + if not isinstance(region, Mapping): + raise ContractValidationError("Evidence region must be an object") + coordinate_space = region.get("coordinate_space") + numeric_coordinates: list[int | float] = [] + for name in ("x", "y", "width", "height"): + number = region.get(name) + if isinstance(number, bool) or not isinstance(number, int | float): + raise ContractValidationError("Evidence bounds must be numeric") + numeric_coordinates.append(number) + x, y, width, height = numeric_coordinates + if coordinate_space == "normalized": + if x < 0 or y < 0 or width <= 0 or height <= 0 or x + width > 1 or y + height > 1: + raise ContractValidationError("Normalized evidence bounds must fit within the image") + elif coordinate_space == "pixels": + if any(not isinstance(number, int) for number in numeric_coordinates): + raise ContractValidationError("Pixel evidence bounds must be integers") + if x < 0 or y < 0 or width <= 0 or height <= 0: + raise ContractValidationError("Pixel evidence bounds must be positive and non-negative") + else: + raise ContractValidationError(f"Unknown coordinate space: {coordinate_space!r}") + + +def _validate_value_state(value: Mapping[str, object], context: str) -> None: + raw_text = value.get("raw_text") + normalized = value.get("normalized_value") + missing_reason = value.get("missing_reason") + if raw_text is not None and not isinstance(raw_text, str): + raise ContractValidationError(f"{context} raw_text must be a string or null") + if normalized is None and not isinstance(missing_reason, str): + raise ContractValidationError(f"Null {context} requires missing_reason") + if normalized is not None and missing_reason is not None: + raise ContractValidationError(f"Present {context} cannot set missing_reason") + + +def _value_state(value: Mapping[str, object], context: str) -> tuple[object, object, object]: + _validate_value_state(value, context) + return value.get("raw_text"), value.get("normalized_value"), value.get("missing_reason") + + +def _parse_correction_time(value: object) -> datetime: + if not isinstance(value, str): + raise ContractValidationError("Correction corrected_at must be a timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ContractValidationError("Correction corrected_at must be a timestamp") from error + if parsed.tzinfo is None: + raise ContractValidationError("Correction corrected_at must include a UTC offset") + return parsed + + +def _validate_correction_history( + evidence: Mapping[str, object], seen_correction_ids: set[str] +) -> None: + history = evidence.get("correction_history") + if not isinstance(history, list): + raise ContractValidationError("correction_history must be an array") + previous_after: tuple[object, object, object] | None = None + previous_time: datetime | None = None + for index, correction in enumerate(history): + if not isinstance(correction, Mapping): + raise ContractValidationError(f"correction_history[{index}] must be an object") + correction_id = correction.get("correction_id") + if not isinstance(correction_id, str): + raise ContractValidationError("correction_id must be a string") + if correction_id in seen_correction_ids: + raise ContractValidationError(f"Duplicate correction_id: {correction_id}") + seen_correction_ids.add(correction_id) + corrected_at = _parse_correction_time(correction.get("corrected_at")) + if previous_time is not None and corrected_at <= previous_time: + raise ContractValidationError("Correction history timestamps must be strictly ordered") + before = correction.get("before") + after = correction.get("after") + if not isinstance(before, Mapping) or not isinstance(after, Mapping): + raise ContractValidationError("Correction before and after must be value states") + before_state = _value_state(before, "correction before") + after_state = _value_state(after, "correction after") + if previous_after is not None and before_state != previous_after: + raise ContractValidationError("Correction history contains a disconnected value state") + previous_after = after_state + previous_time = corrected_at + if previous_after is not None and previous_after != _value_state(evidence, "evidence"): + raise ContractValidationError("Correction history final state does not match evidence") + + +def validate_archive_document(document: Mapping[str, object]) -> None: + validate_archive_references(document) + for screen in _objects(document.get("screens"), "screens"): + classification = screen.get("classification") + if not isinstance(classification, Mapping): + raise ContractValidationError("screen.classification must be an object") + validate_classification(classification) + seen_correction_ids: set[str] = set() + for value in _objects(document.get("evidence_values"), "evidence_values"): + _validate_evidence(value) + _validate_correction_history(value, seen_correction_ids) + + +def validate_handoff_document(document: Mapping[str, object]) -> None: + validate_handoff_references(document) + expected_boundary: dict[str, object] = { + "intended_destination": "authenticated_isac_intake", + "mutation_authority": "none", + "requires_authenticated_action": True, + "preserves_moderation": True, + "creates_audit_events": False, + } + if document.get("trust_boundary") != expected_boundary: + raise ContractValidationError("The handoff trust boundary is invalid") + seen_correction_ids: set[str] = set() + for value in _objects(document.get("evidence_values"), "evidence_values"): + _validate_evidence(value) + _validate_correction_history(value, seen_correction_ids) + + +def validate_archive_references(document: Mapping[str, object]) -> None: + ensure_supported_version(document, SUPPORTED_ARCHIVE_SCHEMA_VERSION) + artifacts = list(_objects(document.get("artifacts"), "artifacts")) + captures = list(_objects(document.get("captures"), "captures")) + screens = list(_objects(document.get("screens"), "screens")) + jobs = list(_objects(document.get("jobs"), "jobs")) + extractions = list(_objects(document.get("extractions"), "extractions")) + groups = list(_objects(document.get("groups"), "groups")) + reviews = list(_objects(document.get("reviews"), "reviews")) + exports = list(_objects(document.get("exports"), "exports")) + errors = list(_objects(document.get("errors"), "errors")) + evidence = list(_objects(document.get("evidence_values"), "evidence_values")) + instances = list(_objects(document.get("observed_instances"), "observed_instances")) + candidates = list(_objects(document.get("canonical_candidates"), "canonical_candidates")) + artifact_ids = _ids(artifacts, "artifact_id") + _ids(captures, "capture_id") + screen_ids = _ids(screens, "screen_id") + _ids(jobs, "job_id") + _ids(extractions, "extraction_id") + group_ids = _ids(groups, "group_id") + _ids(reviews, "review_id") + _ids(exports, "export_id") + error_ids = _ids(errors, "error_id") + evidence_ids = _ids(evidence, "evidence_id") + instance_ids = _ids(instances, "instance_id") + candidate_ids = _ids(candidates, "candidate_id") + artifact_hashes = { + str(artifact.get("artifact_id")): str(artifact.get("sha256")) for artifact in artifacts + } + + for capture in captures: + _require_subset({str(capture.get("artifact_id"))}, artifact_ids, "capture.artifact_id") + for artifact in artifacts: + parent_id = artifact.get("parent_artifact_id") + if parent_id is not None: + if str(parent_id) == str(artifact.get("artifact_id")): + raise ContractValidationError("artifact.parent_artifact_id cannot reference itself") + _require_subset({str(parent_id)}, artifact_ids, "artifact.parent_artifact_id") + for screen in screens: + _require_subset({str(screen.get("artifact_id"))}, artifact_ids, "screen.artifact_id") + for value in evidence: + artifact_id = str(value.get("image_artifact_id")) + _require_subset( + {artifact_id}, + artifact_ids, + "evidence.image_artifact_id", + ) + if artifact_hashes.get(artifact_id) != value.get("image_sha256"): + raise ContractValidationError("Evidence image_sha256 does not match its artifact") + for job in jobs: + last_error_id = job.get("last_error_id") + if last_error_id is not None: + _require_subset({str(last_error_id)}, error_ids, "job.last_error_id") + for extraction in extractions: + _require_subset({str(extraction.get("screen_id"))}, screen_ids, "extraction.screen_id") + _require_subset( + _references(extraction, "evidence_refs"), + evidence_ids, + "extraction.evidence_refs", + ) + for group in groups: + _require_subset(_references(group, "screen_ids"), screen_ids, "group.screen_ids") + supersedes_group_ids = _references(group, "supersedes_group_ids") + if str(group.get("group_id")) in supersedes_group_ids: + raise ContractValidationError("group.supersedes_group_ids cannot reference itself") + _require_subset( + supersedes_group_ids, + group_ids, + "group.supersedes_group_ids", + ) + for instance in instances: + _require_subset( + _references(instance, "source_screen_ids"), + screen_ids, + "observed_instance.source_screen_ids", + ) + for field in _objects(instance.get("fields"), "observed_instance.fields"): + _require_subset( + _references(field, "evidence_refs"), + evidence_ids, + "observed_instance.fields.evidence_refs", + ) + context = instance.get("observed_context") + if not isinstance(context, Mapping): + raise ContractValidationError("observed_instance.observed_context must be an object") + for context_name in ("level", "quality", "tier", "upgrade_state"): + context_field = context.get(context_name) + if not isinstance(context_field, Mapping): + raise ContractValidationError(f"observed_context.{context_name} must be an object") + _require_subset( + _references(context_field, "evidence_refs"), + evidence_ids, + f"observed_context.{context_name}.evidence_refs", + ) + for candidate in candidates: + _require_subset( + _references(candidate, "source_instance_ids"), + instance_ids, + "canonical_candidate.source_instance_ids", + ) + _require_subset( + _references(candidate, "evidence_refs"), + evidence_ids, + "canonical_candidate.evidence_refs", + ) + for field in _objects(candidate.get("fields"), "canonical_candidate.fields"): + _require_subset( + _references(field, "evidence_refs"), + evidence_ids, + "canonical_candidate.fields.evidence_refs", + ) + for review in reviews: + _require_subset({str(review.get("candidate_id"))}, candidate_ids, "review.candidate_id") + for export in exports: + export_artifact_id = export.get("artifact_id") + if export_artifact_id is not None: + _require_subset({str(export_artifact_id)}, artifact_ids, "export.artifact_id") + + +def validate_handoff_references(document: Mapping[str, object]) -> None: + ensure_supported_version(document, SUPPORTED_HANDOFF_SCHEMA_VERSION) + evidence = list(_objects(document.get("evidence_values"), "evidence_values")) + records = list(_objects(document.get("records"), "records")) + evidence_ids = _ids(evidence, "evidence_id") + artifacts = list(_objects(document.get("source_artifacts"), "source_artifacts")) + artifact_ids = _ids(artifacts, "artifact_id") + artifact_hashes = { + str(artifact.get("artifact_id")): str(artifact.get("sha256")) for artifact in artifacts + } + _ids(records, "handoff_record_id") + for value in evidence: + artifact_id = str(value.get("image_artifact_id")) + _require_subset( + {artifact_id}, + artifact_ids, + "evidence.image_artifact_id", + ) + if artifact_hashes.get(artifact_id) != value.get("image_sha256"): + raise ContractValidationError("Evidence image_sha256 does not match its artifact") + for record in records: + for field in _objects(record.get("fields"), "record.fields"): + _require_subset( + _references(field, "evidence_refs"), + evidence_ids, + "record.fields.evidence_refs", + ) + for section_name in ("relationships", "mechanics"): + section = record.get(section_name) + if not isinstance(section, Mapping): + raise ContractValidationError(f"record.{section_name} must be an object") + _require_subset( + _references(section, "evidence_refs"), + evidence_ids, + f"record.{section_name}.evidence_refs", + ) diff --git a/tools/game-data-capture/src/game_data_capture/ingest/__init__.py b/tools/game-data-capture/src/game_data_capture/ingest/__init__.py new file mode 100644 index 0000000..0d7d60d --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/ingest/__init__.py @@ -0,0 +1,4 @@ +"""Wayland clipboard intake and private spool boundaries. + +Submodules stay unimported so the stdin helper can start independently. +""" diff --git a/tools/game-data-capture/src/game_data_capture/ingest/clipboard.py b/tools/game-data-capture/src/game_data_capture/ingest/clipboard.py new file mode 100644 index 0000000..8e8567c --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/ingest/clipboard.py @@ -0,0 +1,510 @@ +"""Bounded, shell-free supervision for Wayland PNG clipboard intake.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import signal +import stat +import subprocess +import sys +import threading +import time +from collections import deque +from collections.abc import Iterator, Mapping +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +from game_data_capture.ingest.spool_helper import ( + PNG_SIGNATURE, + acknowledge_receipt, + durable_receipts, + reconcile_spool, +) + +_SAFE_ENV: Final = frozenset( + {"PATH", "HOME", "LANG", "LC_ALL", "WAYLAND_DISPLAY", "XDG_RUNTIME_DIR"} +) +_DENIED_ENV = re.compile( + r"(?:openai|authorization|token|secret|python(?:home|path|startup|userbase))", re.I +) +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_MAX_METADATA = 16 * 1024 +_MAX_RECEIPT = 1024 + + +def _integer(value: object, minimum: int, maximum: int) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and minimum <= value <= maximum + + +def _finite(value: object, minimum: float, maximum: float) -> bool: + return ( + isinstance(value, int | float) + and not isinstance(value, bool) + and math.isfinite(value) + and minimum < value <= maximum + ) + + +@dataclass(frozen=True, slots=True) +class WatchConfig: + spool: Path + max_image_bytes: int + max_queue_items: int + policy_version: str + max_restarts: int = 3 + restart_backoff_seconds: float = 0.25 + shutdown_timeout_seconds: float = 2.0 + + def __post_init__(self) -> None: + if not self.spool.is_absolute() or len(str(self.spool)) > 240: + raise ValueError("spool must be a bounded absolute path") + if not _integer(self.max_image_bytes, 8, 64 * 1024 * 1024): + raise ValueError("invalid image limit") + if not _integer(self.max_queue_items, 1, 256) or not _integer(self.max_restarts, 0, 10): + raise ValueError("invalid queue or restart limit") + if not _finite(self.restart_backoff_seconds, 0, 60) or not _finite( + self.shutdown_timeout_seconds, 0, 30 + ): + raise ValueError("invalid timeout") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", self.policy_version): + raise ValueError("invalid policy version") + + +@dataclass(frozen=True, slots=True) +class FocusedWindowMetadata: + app_id: str | None + title: str | None + window_id: int | None + + +@dataclass(frozen=True, slots=True) +class IntakeReceipt: + occurrence_id: str + receipt_token: str + sha256: str + byte_size: int + media_type: str + spool_path: Path + policy_version: str + processing_version: str + focused_window: FocusedWindowMetadata + + +class VerifiedSpoolReader: + """Descriptor-verified stream for handing one receipt to GDC-003 safely.""" + + def __init__(self, receipt: IntakeReceipt) -> None: + self.receipt = receipt + self._fd: int | None = None + + def __enter__(self) -> VerifiedSpoolReader: + self._fd = _verify_spool_file(self.receipt) + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def __iter__(self) -> Iterator[bytes]: + if self._fd is None: + raise RuntimeError("verified reader is not open") + while chunk := os.read(self._fd, 64 * 1024): + yield chunk + + def close(self) -> None: + if self._fd is not None: + os.close(self._fd) + self._fd = None + + +def restricted_environment(environment: Mapping[str, str] | None = None) -> dict[str, str]: + source = os.environ if environment is None else environment + return { + key: value[:4096] + for key, value in source.items() + if key in _SAFE_ENV + and not _DENIED_ENV.search(key) + and isinstance(value, str) + and "\x00" not in value + } + + +def _bounded_text(value: object) -> str | None: + return " ".join(value.split())[:256] or None if isinstance(value, str) else None + + +def _bounded_command_output( + arguments: list[str], environment: Mapping[str, str], timeout: float +) -> bytes | None: + try: + process = subprocess.Popen( + arguments, + shell=False, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError: + return None + output = bytearray() + done = threading.Event() + + def collect() -> None: + if process.stdout is not None: + while len(output) <= _MAX_METADATA: + chunk = process.stdout.read(min(4096, _MAX_METADATA + 1 - len(output))) + if not chunk: + break + output.extend(chunk) + done.set() + + reader = threading.Thread(target=collect, daemon=True) + reader.start() + if not done.wait(timeout): + with suppress(OSError): + os.killpg(process.pid, signal.SIGKILL) + with suppress(subprocess.TimeoutExpired): + process.wait(timeout=timeout) + if process.poll() is None: + with suppress(OSError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + if process.stdout is not None: + process.stdout.close() + reader.join(timeout) + return ( + bytes(output) + if done.is_set() and process.returncode == 0 and len(output) <= _MAX_METADATA + else None + ) + + +def focused_window( + *, environment: Mapping[str, str] | None = None, timeout_seconds: float = 0.5 +) -> FocusedWindowMetadata: + if not _finite(timeout_seconds, 0, 30): + return FocusedWindowMetadata(None, None, None) + raw = _bounded_command_output( + ["niri", "msg", "-j", "focused-window"], + restricted_environment(environment), + float(timeout_seconds), + ) + if raw is None: + return FocusedWindowMetadata(None, None, None) + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + return FocusedWindowMetadata(None, None, None) + if not isinstance(payload, dict): + return FocusedWindowMetadata(None, None, None) + identifier = payload.get("id") + return FocusedWindowMetadata( + _bounded_text(payload.get("app_id")), + _bounded_text(payload.get("title")), + identifier if _integer(identifier, 0, 2**63 - 1) else None, + ) + + +def _verify_spool_file(receipt: IntakeReceipt) -> int: + spool = receipt.spool_path.parent + if spool.is_symlink() or not spool.is_dir() or (spool.stat().st_mode & 0o777) != 0o700: + raise ValueError("unsafe spool") + fd = os.open(receipt.spool_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + info = os.fstat(fd) + if ( + not stat.S_ISREG(info.st_mode) + or (info.st_mode & 0o777) != 0o600 + or info.st_size != receipt.byte_size + ): + raise ValueError("invalid spool file") + first = os.read(fd, len(PNG_SIGNATURE)) + if first != PNG_SIGNATURE: + raise ValueError("invalid spool png") + hasher = hashlib.sha256(first) + while chunk := os.read(fd, 64 * 1024): + hasher.update(chunk) + if hasher.hexdigest() != receipt.sha256: + raise ValueError("spool digest mismatch") + os.lseek(fd, 0, os.SEEK_SET) + return fd + except Exception: + os.close(fd) + raise + + +class ClipboardIntake: + def __init__( + self, config: WatchConfig, *, environment: Mapping[str, str] | None = None + ) -> None: + self.config, self.environment = config, restricted_environment(environment) + self.armed = self.paused = False + self._watcher: subprocess.Popen[bytes] | None = None + self._reader: threading.Thread | None = None + self._queue: deque[IntakeReceipt] = deque() + self._helper_payloads: deque[dict[str, object]] = deque() + self._seen_tokens: set[str] = set() + self._seen_order: deque[str] = deque() + self._recovered_tokens: set[str] = set() + self._restart_count = 0 + self._restart_after = 0.0 + self._restarts_exhausted = False + + @property + def queue_depth(self) -> int: + return len(self._queue) + + @property + def helper_queue_depth(self) -> int: + return len(self._helper_payloads) + + def arm(self) -> None: + self.armed, self.paused = True, False + self._start_if_due(time.monotonic()) + + def pause(self) -> None: + self.paused = True + self._stop_watcher() + + def resume(self) -> None: + if self.armed: + self.paused = False + self._start_if_due(time.monotonic()) + + def disarm(self) -> None: + self.armed = self.paused = False + self._stop_watcher() + + def cancel(self) -> None: + self.shutdown() + + def shutdown(self) -> None: + self.disarm() + self._queue.clear() + self._helper_payloads.clear() + self._seen_tokens.clear() + self._seen_order.clear() + self._recovered_tokens.clear() + + def tick(self, now: float | None = None) -> None: + current = time.monotonic() if now is None else now + if self._watcher is not None and self._watcher.poll() is not None: + self._stop_watcher() + self._crashed(current) + self._start_if_due(current) + + def take(self) -> IntakeReceipt | None: + return self._queue.popleft() if self._queue else None + + def take_helper_receipt(self) -> dict[str, object] | None: + return self._helper_payloads.popleft() if self._helper_payloads else None + + def unreferenced_spool_candidates(self) -> list[Path]: + queued = {item.spool_path.name for item in self._queue} + return [ + path + for path in self.config.spool.glob("*.png") + if not path.is_symlink() and path.name not in queued + ] + + def open_verified(self, receipt: IntakeReceipt) -> VerifiedSpoolReader: + return VerifiedSpoolReader(receipt) + + def accept_helper_receipt( + self, + payload: Mapping[str, object], + *, + occurrence_id: str, + processing_version: str, + metadata: FocusedWindowMetadata | None = None, + ) -> bool: + token, digest, size, name = ( + payload.get("receipt_token"), + payload.get("sha256"), + payload.get("byte_size"), + payload.get("spool_name"), + ) + if ( + not self.armed + or self.paused + or len(self._queue) >= self.config.max_queue_items + or payload.get("receipt_version") != 1 + or not isinstance(token, str) + or not _ID.fullmatch(token) + or token in self._seen_tokens + or not isinstance(digest, str) + or not _SHA256.fullmatch(digest) + or not _integer(size, 8, self.config.max_image_bytes) + or payload.get("media_type") != "image/png" + or payload.get("policy_version") != self.config.policy_version + or not isinstance(name, str) + or name != f"{digest}.png" + or not isinstance(occurrence_id, str) + or not _ID.fullmatch(occurrence_id) + or not isinstance(processing_version, str) + or not _ID.fullmatch(processing_version) + ): + return False + recovered = token in self._recovered_tokens + receipt = IntakeReceipt( + occurrence_id, + token, + digest, + cast(int, size), + "image/png", + self.config.spool / name, + self.config.policy_version, + processing_version, + FocusedWindowMetadata(None, None, None) + if recovered + else focused_window() + if metadata is None + else metadata, + ) + try: + verified_fd = _verify_spool_file(receipt) + except (OSError, ValueError): + return False + os.close(verified_fd) + self._seen_tokens.add(token) + self._seen_order.append(token) + if len(self._seen_order) > self.config.max_queue_items: + self._seen_tokens.remove(self._seen_order.popleft()) + self._queue.append(receipt) + return True + + def reconcile(self) -> list[str]: + removed = reconcile_spool(self.config.spool) + capacity = self.config.max_queue_items - len(self._helper_payloads) + if capacity <= 0: + return removed + for payload in durable_receipts(self.config.spool, max_items=256): + if len(self._helper_payloads) >= self.config.max_queue_items: + break + token = payload["receipt_token"] + if isinstance(token, str) and token not in self._seen_tokens: + self._helper_payloads.append(payload) + self._recovered_tokens.add(token) + return removed + + def acknowledge(self, receipt: IntakeReceipt) -> bool: + """Delete this occurrence's durable receipt after repository confirmation only.""" + if ( + receipt.spool_path.parent != self.config.spool + or receipt.policy_version != self.config.policy_version + ): + return False + payload: dict[str, object] = { + "receipt_version": 1, + "receipt_token": receipt.receipt_token, + "sha256": receipt.sha256, + "byte_size": receipt.byte_size, + "media_type": receipt.media_type, + "spool_name": receipt.spool_path.name, + "policy_version": receipt.policy_version, + } + acknowledged = acknowledge_receipt(self.config.spool, payload) + if acknowledged: + self._recovered_tokens.discard(receipt.receipt_token) + return acknowledged + + def _crashed(self, now: float) -> None: + if self.armed and not self.paused and self._restart_count < self.config.max_restarts: + self._restart_count += 1 + self._restart_after = now + float(self.config.restart_backoff_seconds) * 2 ** ( + self._restart_count - 1 + ) + else: + self._restarts_exhausted = True + + def _start_if_due(self, now: float) -> None: + if ( + not self.armed + or self.paused + or self._watcher is not None + or self._restarts_exhausted + or now < self._restart_after + ): + return + try: + self._watcher = subprocess.Popen( + [ + "wl-paste", + "--type", + "image/png", + "--watch", + sys.executable, + "-m", + "game_data_capture.ingest.spool_helper", + "--spool", + str(self.config.spool), + "--max-bytes", + str(self.config.max_image_bytes), + "--policy-version", + self.config.policy_version, + "--media-type", + "image/png", + ], + shell=False, + env=self.environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError: + self._crashed(now) + return + self._reader = threading.Thread( + target=self._read_receipts, args=(self._watcher,), daemon=True + ) + self._reader.start() + + def _read_receipts(self, process: subprocess.Popen[bytes]) -> None: + if process.stdout is None: + return + while True: + line = process.stdout.readline(_MAX_RECEIPT + 1) + if not line: + return + if len(line) > _MAX_RECEIPT and not line.endswith(b"\n"): + while line and not line.endswith(b"\n"): + line = process.stdout.readline(_MAX_RECEIPT + 1) + continue + try: + payload = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + continue + if ( + isinstance(payload, dict) + and all(isinstance(key, str) for key in payload) + and len(self._helper_payloads) < self.config.max_queue_items + ): + self._helper_payloads.append(payload) + + def _stop_watcher(self) -> None: + process, self._watcher = self._watcher, None + if process is None: + return + if process.poll() is None: + with suppress(OSError): + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=float(self.config.shutdown_timeout_seconds)) + except subprocess.TimeoutExpired: + with suppress(OSError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + if process.stdout is not None: + process.stdout.close() + if self._reader is not None: + self._reader.join(timeout=float(self.config.shutdown_timeout_seconds)) + self._reader = None diff --git a/tools/game-data-capture/src/game_data_capture/ingest/spool_helper.py b/tools/game-data-capture/src/game_data_capture/ingest/spool_helper.py new file mode 100644 index 0000000..217901b --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/ingest/spool_helper.py @@ -0,0 +1,486 @@ +"""Small stdin-only PNG spool helper; intentionally independent of application services.""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import os +import re +import secrets +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager, suppress +from pathlib import Path +from stat import S_ISDIR, S_ISREG + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +RECEIPT_VERSION = 1 +_TOKEN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CHUNK_SIZE = 64 * 1024 +_MAX_RECEIPT_BYTES = 1024 +_MAX_SIDECARS = 256 +_MAX_RECONCILE_ENTRIES = 1024 +_LOCK_NAME = ".spool.lock" +_LOCK_TIMEOUT_SECONDS = 1.0 +_LOCK_RETRY_SECONDS = 0.01 + + +class SpoolError(ValueError): + """The supplied image or spool boundary is invalid.""" + + +def _private_spool(path: Path) -> None: + if not path.is_absolute() or len(str(path)) > 240: + raise SpoolError("invalid spool") + if path.is_symlink(): + raise SpoolError("unsafe spool") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + mode = path.lstat().st_mode + if path.is_symlink() or not S_ISDIR(mode): + raise SpoolError("unsafe spool") + path.chmod(0o700) + + +def _write_all(fd: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(fd, view) + if written <= 0: + raise OSError("short spool write") + view = view[written:] + + +def _fsync_directory(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +@contextmanager +def _spool_lock(spool: Path) -> Iterator[None]: + """Hold the private spool lock for one bounded receipt installation.""" + try: + fd = os.open(spool / _LOCK_NAME, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + except OSError as error: + raise SpoolError("unsafe spool lock") from error + try: + info = os.fstat(fd) + if not S_ISREG(info.st_mode) or (info.st_mode & 0o777) != 0o600: + raise SpoolError("unsafe spool lock") + deadline = time.monotonic() + _LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise SpoolError("spool busy") from None + time.sleep(_LOCK_RETRY_SECONDS) + yield + finally: + with suppress(OSError): + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _verify_existing(path: Path, digest: str, size: int) -> bool: + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except OSError: + return False + try: + info = os.fstat(fd) + if not S_ISREG(info.st_mode) or (info.st_mode & 0o777) != 0o600 or info.st_size != size: + return False + hasher = hashlib.sha256() + prefix = os.read(fd, len(PNG_SIGNATURE)) + if prefix != PNG_SIGNATURE: + return False + hasher.update(prefix) + while chunk := os.read(fd, _CHUNK_SIZE): + hasher.update(chunk) + return hasher.hexdigest() == digest + finally: + os.close(fd) + + +def receipt_name(token: str) -> str: + if not _TOKEN.fullmatch(token): + raise SpoolError("invalid receipt identity") + return f"{token}.receipt.json" + + +def _pending_name(token: str) -> str: + return f".{token}.receipt.pending" + + +def _receipt_bytes(receipt: dict[str, object]) -> bytes: + encoded = json.dumps(receipt, separators=(",", ":"), sort_keys=True).encode("ascii") + if len(encoded) > _MAX_RECEIPT_BYTES: + raise SpoolError("receipt exceeds limit") + return encoded + + +def _valid_receipt(payload: object, *, spool: Path, verify_png: bool) -> dict[str, object] | None: + if not isinstance(payload, dict) or set(payload) != { + "byte_size", + "media_type", + "policy_version", + "receipt_token", + "receipt_version", + "sha256", + "spool_name", + }: + return None + token, digest, size, name = ( + payload.get("receipt_token"), + payload.get("sha256"), + payload.get("byte_size"), + payload.get("spool_name"), + ) + if ( + payload.get("receipt_version") != RECEIPT_VERSION + or not isinstance(token, str) + or not _TOKEN.fullmatch(token) + or not isinstance(digest, str) + or not _SHA256.fullmatch(digest) + or isinstance(size, bool) + or not isinstance(size, int) + or size < len(PNG_SIGNATURE) + or payload.get("media_type") != "image/png" + or not isinstance(payload.get("policy_version"), str) + or not _TOKEN.fullmatch(payload["policy_version"]) + or not isinstance(name, str) + or name != f"{digest}.png" + ): + return None + if verify_png and not _verify_existing(spool / name, digest, size): + return None + return payload + + +def _read_sidecar(path: Path, *, spool: Path, verify_png: bool) -> dict[str, object] | None: + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except OSError: + return None + try: + info = os.fstat(fd) + if ( + not S_ISREG(info.st_mode) + or (info.st_mode & 0o777) != 0o600 + or info.st_size > _MAX_RECEIPT_BYTES + ): + return None + raw = os.read(fd, _MAX_RECEIPT_BYTES + 1) + if len(raw) != info.st_size or len(raw) > _MAX_RECEIPT_BYTES: + return None + try: + return _valid_receipt(json.loads(raw), spool=spool, verify_png=verify_png) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + finally: + os.close(fd) + + +def _install_pending_sidecar(spool: Path, receipt: dict[str, object]) -> Path: + token = receipt["receipt_token"] + assert isinstance(token, str) + pending = spool / _pending_name(token) + encoded = _receipt_bytes(receipt) + fd = os.open(pending, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + _write_all(fd, encoded) + os.fsync(fd) + finally: + os.close(fd) + # Persist the recoverable pending name before promotion. A crash after this + # point cannot claim recovery until its PNG is immutably installed. + _fsync_directory(spool) + return pending + + +def _same_receipt(left: dict[str, object], right: dict[str, object]) -> bool: + return _receipt_bytes(left) == _receipt_bytes(right) + + +def _final_sidecar(spool: Path, receipt: dict[str, object]) -> dict[str, object] | None: + token = receipt["receipt_token"] + assert isinstance(token, str) + final = spool / receipt_name(token) + if not final.exists() and not final.is_symlink(): + return None + stored = _read_sidecar(final, spool=spool, verify_png=True) + if stored is None or not _same_receipt(stored, receipt): + raise SpoolError("receipt token conflict") + return stored + + +def _receipt_capacity_available(spool: Path) -> bool: + """Count bounded valid receipt sidecars while removing malformed regular entries.""" + valid = 0 + candidates = 0 + for entry in spool.iterdir(): + if entry.is_symlink() or not entry.is_file(): + continue + pending = entry.name.startswith(".") and entry.name.endswith(".receipt.pending") + final = entry.name.endswith(".receipt.json") + if not pending and not final: + continue + candidates += 1 + if candidates > _MAX_RECONCILE_ENTRIES: + return False + token = ( + entry.name[1 : -len(".receipt.pending")] + if pending + else entry.name[: -len(".receipt.json")] + ) + receipt = _read_sidecar(entry, spool=spool, verify_png=final) + if receipt is None or receipt.get("receipt_token") != token: + entry.unlink() + _fsync_directory(spool) + continue + valid += 1 + if valid >= _MAX_SIDECARS: + return False + return True + + +def _promote_sidecar(spool: Path, receipt: dict[str, object], pending: Path) -> None: + token = receipt["receipt_token"] + assert isinstance(token, str) + final = spool / receipt_name(token) + try: + os.link(pending, final, follow_symlinks=False) + except FileExistsError: + try: + _final_sidecar(spool, receipt) + except SpoolError: + pending.unlink() + _fsync_directory(spool) + raise + else: + _fsync_directory(spool) + pending.unlink() + _fsync_directory(spool) + + +def reconcile_spool(spool: Path) -> list[str]: + """Remove incomplete files and promote only receipt sidecars backed by immutable PNGs.""" + _private_spool(spool) + removed: list[str] = [] + promoted = 0 + candidates = 0 + for entry in spool.iterdir(): + if entry.is_symlink() or not entry.is_file(): + continue + if entry.name.endswith(".partial"): + candidates += 1 + if candidates > _MAX_RECONCILE_ENTRIES: + break + entry.unlink() + removed.append(entry.name) + elif entry.name.startswith(".") and entry.name.endswith(".receipt.pending"): + candidates += 1 + if candidates > _MAX_RECONCILE_ENTRIES: + break + receipt = _read_sidecar(entry, spool=spool, verify_png=True) + token = entry.name[1 : -len(".receipt.pending")] + if receipt is not None and receipt.get("receipt_token") == token: + final = spool / receipt_name(token) + if final.exists() or final.is_symlink(): + with suppress(SpoolError): + _final_sidecar(spool, receipt) + entry.unlink() + removed.append(entry.name) + elif promoted >= _MAX_SIDECARS: + continue + else: + _promote_sidecar(spool, receipt, entry) + promoted += 1 + continue + entry.unlink() + removed.append(entry.name) + return removed + + +def durable_receipts(spool: Path, *, max_items: int) -> Iterator[dict[str, object]]: + """Yield at most ``max_items`` validated final receipt sidecars without following links.""" + if ( + isinstance(max_items, bool) + or not isinstance(max_items, int) + or not 1 <= max_items <= _MAX_SIDECARS + ): + raise SpoolError("invalid receipt limit") + _private_spool(spool) + yielded = 0 + valid = 0 + candidates = 0 + for entry in spool.iterdir(): + if entry.is_symlink() or not entry.is_file() or not entry.name.endswith(".receipt.json"): + continue + candidates += 1 + if candidates > _MAX_RECONCILE_ENTRIES: + break + token = entry.name[: -len(".receipt.json")] + receipt = _read_sidecar(entry, spool=spool, verify_png=True) + if receipt is not None and receipt.get("receipt_token") == token: + valid += 1 + if valid > _MAX_SIDECARS: + break + if yielded < max_items: + yielded += 1 + yield receipt + else: + try: + info = entry.lstat() + if S_ISREG(info.st_mode): + entry.unlink() + _fsync_directory(spool) + except OSError: + continue + + +def acknowledge_receipt(spool: Path, receipt: dict[str, object]) -> bool: + """Remove exactly one verified durable sidecar; immutable PNG data is never removed here.""" + _private_spool(spool) + token = receipt.get("receipt_token") + if not isinstance(token, str) or not _TOKEN.fullmatch(token): + return False + path = spool / receipt_name(token) + stored = _read_sidecar(path, spool=spool, verify_png=True) + if stored is None: + return False + if _receipt_bytes(stored) != _receipt_bytes(receipt): + return False + try: + info = path.lstat() + if not S_ISREG(info.st_mode) or (info.st_mode & 0o777) != 0o600: + return False + path.unlink() + _fsync_directory(spool) + except OSError: + return False + return True + + +def spool_png( + input_stream: object, + *, + spool: Path, + max_bytes: int, + receipt_token: str, + policy_version: str, + media_type: str = "image/png", +) -> dict[str, object]: + """Install immutable PNG bytes, then fsync a durable occurrence receipt before stdout.""" + if ( + isinstance(max_bytes, bool) + or not isinstance(max_bytes, int) + or not 8 <= max_bytes <= 64 * 1024 * 1024 + ): + raise SpoolError("invalid byte limit") + if ( + media_type != "image/png" + or not _TOKEN.fullmatch(receipt_token) + or not _TOKEN.fullmatch(policy_version) + ): + raise SpoolError("invalid receipt identity") + _private_spool(spool) + temporary = spool / f".{secrets.token_hex(16)}.partial" + pending: Path | None = None + immutable_installed = False + hasher = hashlib.sha256() + size = 0 + try: + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + prefix = bytearray() + while True: + chunk = input_stream.read(_CHUNK_SIZE) # type: ignore[attr-defined] + if not isinstance(chunk, bytes): + raise SpoolError("invalid input stream") + if not chunk: + break + prefix.extend(chunk[: max(0, len(PNG_SIGNATURE) - len(prefix))]) + size += len(chunk) + if size > max_bytes: + raise SpoolError("image exceeds byte limit") + hasher.update(chunk) + _write_all(fd, chunk) + if len(prefix) < len(PNG_SIGNATURE) or bytes(prefix) != PNG_SIGNATURE: + raise SpoolError("invalid png") + os.fsync(fd) + finally: + os.close(fd) + digest = hasher.hexdigest() + name = f"{digest}.png" + receipt: dict[str, object] = { + "receipt_version": RECEIPT_VERSION, + "receipt_token": receipt_token, + "sha256": digest, + "byte_size": size, + "media_type": "image/png", + "spool_name": name, + "policy_version": policy_version, + } + with _spool_lock(spool): + if _final_sidecar(spool, receipt) is not None: + return receipt + if not _receipt_capacity_available(spool): + raise SpoolError("spool receipt capacity reached") + pending = _install_pending_sidecar(spool, receipt) + destination = spool / name + if destination.exists(): + if not _verify_existing(destination, digest, size): + raise SpoolError("spool collision") + else: + os.replace(temporary, destination) + os.chmod(destination, 0o600) + _fsync_directory(spool) + immutable_installed = True + _promote_sidecar(spool, receipt, pending) + return receipt + finally: + if temporary.exists() and not temporary.is_symlink(): + temporary.unlink() + if ( + pending is not None + and not immutable_installed + and pending.exists() + and not pending.is_symlink() + ): + pending.unlink() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--spool", required=True) + parser.add_argument("--max-bytes", required=True, type=int) + parser.add_argument("--receipt-token", default=secrets.token_hex(16)) + parser.add_argument("--policy-version", required=True) + parser.add_argument("--media-type", default="image/png") + args = parser.parse_args(argv) + try: + receipt = spool_png( + sys.stdin.buffer, + spool=Path(args.spool), + max_bytes=args.max_bytes, + receipt_token=args.receipt_token, + policy_version=args.policy_version, + media_type=args.media_type, + ) + except (OSError, SpoolError): + return 2 + sys.stdout.write(json.dumps(receipt, separators=(",", ":")) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/game-data-capture/src/game_data_capture/logging.py b/tools/game-data-capture/src/game_data_capture/logging.py new file mode 100644 index 0000000..f247bf3 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/logging.py @@ -0,0 +1,95 @@ +"""Structured logging helpers that remove sensitive capture material.""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Mapping +from typing import Any + +_SENSITIVE_FIELD = re.compile( + r"(?:api[_-]?key|authorization|auth[_-]?header|token|secret|password|" + r"image|payload|base64|data[_-]?url|ocr[_-]?text|recognized[_-]?text)", + re.IGNORECASE, +) +_AUTHORIZATION_VALUE = re.compile(r"(?i)(authorization\s*[:=]\s*)(?:bearer\s+)?\S+") +_KEY_VALUE = re.compile(r"(?i)((?:api[_-]?key|token|secret|password)\s*[:=]\s*)\S+") +_OPENAI_KEY = re.compile(r"\bsk-[A-Za-z0-9_-]{8,}\b") + + +def redact_text(value: str) -> str: + """Remove common secret-bearing fragments from a log message.""" + + value = _AUTHORIZATION_VALUE.sub(r"\1[REDACTED]", value) + value = _KEY_VALUE.sub(r"\1[REDACTED]", value) + return _OPENAI_KEY.sub("[REDACTED]", value) + + +def redact_log_data(value: object, field_name: str | None = None) -> object: + """Redact secrets, image data, and unbounded OCR text from structured fields.""" + + if field_name is not None and _SENSITIVE_FIELD.search(field_name): + return "[REDACTED]" + if isinstance(value, Mapping): + return {str(key): redact_log_data(item, str(key)) for key, item in value.items()} + if isinstance(value, list | tuple | set | frozenset): + return [redact_log_data(item) for item in value] + if isinstance(value, str): + return redact_text(value[:512]) + return value + + +class RedactingFormatter(logging.Formatter): + """Formatter for static messages plus redacted structured event data.""" + + def format(self, record: logging.LogRecord) -> str: + original_message = record.msg + original_args = record.args + original_exception = record.exc_info + original_exception_text = record.exc_text + original_stack = record.stack_info + try: + if isinstance(record.msg, str): + record.msg = redact_text(record.msg) + if isinstance(record.args, Mapping): + record.args = { + str(key): redact_log_data(value, str(key)) for key, value in record.args.items() + } + elif record.args: + record.args = tuple(redact_log_data(item) for item in record.args) + record.exc_info = None + record.exc_text = None + record.stack_info = None + output = redact_text(super().format(record)) + if original_exception is not None or original_exception_text is not None: + return f"{output}\n[REDACTED EXCEPTION]" + return output + finally: + record.msg = original_message + record.args = original_args + record.exc_info = original_exception + record.exc_text = original_exception_text + record.stack_info = original_stack + + +def configure_logging(level: int = logging.INFO) -> logging.Logger: + """Create an isolated logger that writes redacted records to stderr.""" + + logger = logging.getLogger("game_data_capture") + logger.setLevel(level) + logger.propagate = False + for handler in logger.handlers[:]: + logger.removeHandler(handler) + handler = logging.StreamHandler() + handler.setFormatter(RedactingFormatter("%(levelname)s %(message)s")) + logger.addHandler(handler) + return logger + + +def log_event(logger: logging.Logger, event: str, **fields: Any) -> None: + """Log a stable event name and scrubbed structured fields. + + Callers must keep event names static and place external text in named fields. + """ + + logger.info("%s %s", event, redact_log_data(fields)) diff --git a/tools/game-data-capture/src/game_data_capture/paths.py b/tools/game-data-capture/src/game_data_capture/paths.py new file mode 100644 index 0000000..e0f240a --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/paths.py @@ -0,0 +1,108 @@ +"""XDG path resolution and private-directory preparation. + +Path resolution is pure. Call :func:`prepare_private_directories` only at a runtime +boundary that is allowed to create the utility's private directories. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from os import environ +from pathlib import Path +from stat import S_ISDIR, S_ISLNK, S_ISREG + + +class PathConfigurationError(ValueError): + """A configured path violates the local runtime contract.""" + + +def _bounded_path_error(kind: str) -> PathConfigurationError: + return PathConfigurationError(f"{kind} path must be an absolute path within 240 characters") + + +def _read_xdg_path(environment: Mapping[str, str], key: str, fallback: Path, kind: str) -> Path: + raw_value = environment.get(key) + candidate = fallback if raw_value is None or raw_value == "" else Path(raw_value).expanduser() + if not candidate.is_absolute() or len(str(candidate)) > 240: + raise _bounded_path_error(kind) + return candidate + + +@dataclass(frozen=True, slots=True) +class RuntimePaths: + """Private filesystem locations owned by one local capture installation.""" + + config_dir: Path + data_dir: Path + cache_dir: Path + runtime_dir: Path + + def __post_init__(self) -> None: + for path in (self.config_dir, self.data_dir, self.cache_dir, self.runtime_dir): + if not path.is_absolute() or len(str(path)) > 240: + raise _bounded_path_error("runtime") + + @property + def config_file(self) -> Path: + return self.config_dir / "config.toml" + + @property + def instance_lock_file(self) -> Path: + return self.runtime_dir / "instance.lock" + + +def resolve_runtime_paths(environment: Mapping[str, str] | None = None) -> RuntimePaths: + """Resolve XDG paths without creating or inspecting filesystem entries.""" + + env = environ if environment is None else environment + home = Path(env.get("HOME", "~")).expanduser() + if not home.is_absolute() or len(str(home)) > 200: + raise _bounded_path_error("home") + + config_root = _read_xdg_path(env, "XDG_CONFIG_HOME", home / ".config", "XDG config") + data_root = _read_xdg_path(env, "XDG_DATA_HOME", home / ".local" / "share", "XDG data") + cache_root = _read_xdg_path(env, "XDG_CACHE_HOME", home / ".cache", "XDG cache") + runtime_root = _read_xdg_path(env, "XDG_RUNTIME_DIR", data_root / "runtime", "XDG runtime") + + return RuntimePaths( + config_dir=config_root / "shd-capture", + data_dir=data_root / "shd-capture", + cache_dir=cache_root / "shd-capture", + runtime_dir=runtime_root / "shd-capture", + ) + + +def _ensure_private_directory(path: Path) -> None: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + mode = path.lstat().st_mode + if S_ISLNK(mode) or not S_ISDIR(mode): + raise PathConfigurationError("runtime path must name a directory") + path.chmod(0o700) + + +def prepare_private_directories(paths: RuntimePaths) -> None: + """Create utility directories with owner-only access at an explicit runtime boundary.""" + + for path in (paths.config_dir, paths.data_dir, paths.cache_dir, paths.runtime_dir): + _ensure_private_directory(path) + + +def prepare_private_file(path: Path) -> None: + """Create a runtime-owned regular file with owner-only access. + + The lifecycle layer uses this contract for its instance lock and any future + state files. Callers must prepare the parent directory first. + """ + + if not path.is_absolute() or len(str(path)) > 240: + raise _bounded_path_error("runtime") + if path.is_symlink(): + raise PathConfigurationError("runtime file must be a regular file") + if path.exists(): + mode = path.lstat().st_mode + if S_ISLNK(mode) or not S_ISREG(mode): + raise PathConfigurationError("runtime file must be a regular file") + else: + path.touch(mode=0o600, exist_ok=False) + path.chmod(0o600) diff --git a/tools/game-data-capture/src/game_data_capture/pipeline/jobs.py b/tools/game-data-capture/src/game_data_capture/pipeline/jobs.py new file mode 100644 index 0000000..2a3f27d --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/pipeline/jobs.py @@ -0,0 +1,157 @@ +"""Bounded durable leases that follow the frozen job state machine.""" + +from __future__ import annotations + +import sqlite3 +from datetime import UTC, datetime, timedelta + +from game_data_capture.domain.state_machines import require_transition +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import StorageError, _id + +_KINDS = {"classify", "ocr", "vision_fallback", "frame_extract", "export"} + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _lease(seconds: int) -> None: + if isinstance(seconds, bool) or not isinstance(seconds, int) or not 1 <= seconds <= 300: + raise ValueError("lease duration must be from 1 to 300 seconds") + + +class JobLeases: + def __init__(self, database: StorageDatabase) -> None: + self.database = database + + def create( + self, job_id: str, kind: str, capture_id: str | None = None, max_attempts: int = 3 + ) -> None: + _id(job_id) + if capture_id is not None: + _id(capture_id) + if ( + kind not in _KINDS + or isinstance(max_attempts, bool) + or not isinstance(max_attempts, int) + or not 1 <= max_attempts <= 20 + ): + raise ValueError("invalid job definition") + with self.database.connect() as connection: + connection.execute( + "INSERT INTO jobs(job_id, capture_id, kind, state, max_attempts) " + "VALUES (?, ?, ?, 'queued', ?)", + (job_id, capture_id, kind, max_attempts), + ) + + def _recover_expired(self, connection: sqlite3.Connection, now: datetime) -> None: + # State changes use the contract path: running -> retry_wait -> queued/failed. + rows = list( + connection.execute( + "SELECT job_id, attempt, max_attempts FROM jobs " + "WHERE state = 'running' AND lease_expires_at < ?", + (now.isoformat(),), + ) + ) + for row in rows: + require_transition("job", "running", "retry_wait") + changed = connection.execute( + "UPDATE jobs SET state = 'retry_wait', lease_owner = NULL, " + "lease_expires_at = NULL, revision = revision + 1 " + "WHERE job_id = ? AND state = 'running'", + (row["job_id"],), + ).rowcount + if changed != 1: + raise StorageError("expired job changed concurrently") + target = "failed" if row["attempt"] >= row["max_attempts"] else "queued" + require_transition("job", "retry_wait", target) + changed = connection.execute( + "UPDATE jobs SET state = ?, revision = revision + 1 " + "WHERE job_id = ? AND state = 'retry_wait'", + (target, row["job_id"]), + ).rowcount + if changed != 1: + raise StorageError("expired job changed concurrently") + + def acquire(self, owner: str, lease_seconds: int) -> str | None: + _id(owner) + _lease(lease_seconds) + now = _now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._recover_expired(connection, now) + row = connection.execute( + "SELECT job_id FROM jobs WHERE state = 'queued' " + "AND attempt < max_attempts ORDER BY job_id LIMIT 1" + ).fetchone() + if row is None: + connection.execute("COMMIT") + return None + result = connection.execute( + "UPDATE jobs SET state = 'running', lease_owner = ?, lease_expires_at = ?, " + "attempt = attempt + 1, revision = revision + 1 WHERE job_id = ? " + "AND state = 'queued' AND attempt < max_attempts", + (owner, (now + timedelta(seconds=lease_seconds)).isoformat(), row["job_id"]), + ) + connection.execute("COMMIT") + return str(row["job_id"]) if result.rowcount == 1 else None + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + + def renew(self, job_id: str, owner: str, lease_seconds: int) -> None: + _id(job_id) + _id(owner) + _lease(lease_seconds) + now = _now() + with self.database.connect() as connection: + result = connection.execute( + "UPDATE jobs SET lease_expires_at = ?, revision = revision + 1 " + "WHERE job_id = ? AND state = 'running' AND lease_owner = ? " + "AND lease_expires_at >= ?", + ( + (now + timedelta(seconds=lease_seconds)).isoformat(), + job_id, + owner, + now.isoformat(), + ), + ) + if result.rowcount != 1: + raise StorageError("lease is not owned or has expired") + + def finish(self, job_id: str, owner: str, target: str) -> None: + _id(job_id) + _id(owner) + if target not in {"succeeded", "failed", "cancelled"}: + raise ValueError("job finish target must be terminal") + require_transition("job", "running", target) + now = _now() + with self.database.connect() as connection: + result = connection.execute( + "UPDATE jobs SET state = ?, lease_owner = NULL, lease_expires_at = NULL, " + "revision = revision + 1 WHERE job_id = ? AND state = 'running' " + "AND lease_owner = ? AND lease_expires_at >= ?", + (target, job_id, owner, now.isoformat()), + ) + if result.rowcount != 1: + raise StorageError("lease is not owned or has expired") + + def recover(self) -> int: + now = _now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + before = connection.execute( + "SELECT count(*) FROM jobs WHERE state = 'running' AND lease_expires_at < ?", + (now.isoformat(),), + ).fetchone()[0] + self._recover_expired(connection, now) + connection.execute("COMMIT") + return int(before) + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise diff --git a/tools/game-data-capture/src/game_data_capture/runtime.py b/tools/game-data-capture/src/game_data_capture/runtime.py new file mode 100644 index 0000000..c20c1f9 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/runtime.py @@ -0,0 +1,545 @@ +"""Bounded lifecycle wiring for local capture intake and utility storage.""" + +from __future__ import annotations + +import fcntl +import os +import secrets +import signal +import stat +import time +from collections.abc import Callable, Iterable, Iterator, Mapping +from contextlib import AbstractContextManager, contextmanager, suppress +from dataclasses import dataclass +from pathlib import Path +from threading import Event, RLock, Thread, current_thread, main_thread +from typing import Protocol, cast + +import uvicorn + +from game_data_capture.config import AppConfig +from game_data_capture.ingest.clipboard import ClipboardIntake, IntakeReceipt, WatchConfig +from game_data_capture.ingest.spool_helper import PNG_SIGNATURE +from game_data_capture.paths import RuntimePaths, prepare_private_directories, prepare_private_file +from game_data_capture.pipeline.jobs import JobLeases +from game_data_capture.storage import ( + CaptureCorpus, + CaptureRepository, + ContentAddressedBlobStore, + StorageDatabase, +) +from game_data_capture.web.app import ( + CaptureControlPort, + CaptureSessionStatus, + ControlOutcome, + WebConfig, + create_app, +) +from game_data_capture.web.security import new_bootstrap_token + +_POLICY_VERSION = "capture-v1" +_PROCESSING_VERSION = "capture-v1" +_POLL_SECONDS = 0.1 + + +class RuntimeFailure(RuntimeError): + """A safe-to-display runtime failure that does not disclose local inputs.""" + + +class IntakePort(Protocol): + """The accepted media descriptor boundary consumed by this runtime.""" + + def reconcile(self) -> list[str]: ... + + def arm(self) -> None: ... + + def tick(self) -> None: ... + + def take_helper_receipt(self) -> dict[str, object] | None: ... + + def accept_helper_receipt( + self, + payload: Mapping[str, object], + *, + occurrence_id: str, + processing_version: str, + ) -> bool: ... + + def take(self) -> IntakeReceipt | None: ... + + def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: ... + + def acknowledge(self, receipt: IntakeReceipt) -> bool: ... + + def shutdown(self) -> None: ... + + +class InstanceLock: + """Descriptor-held ownership that safely reuses an unlocked stale lock file.""" + + def __init__(self, file_descriptor: int) -> None: + self._file_descriptor: int | None = file_descriptor + + @classmethod + def acquire(cls, path: Path, timeout_seconds: int) -> InstanceLock: + try: + prepare_private_file(path) + descriptor = os.open(path, os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as error: + raise RuntimeFailure("capture instance lock is unavailable") from error + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600: + raise RuntimeFailure("capture instance lock is unsafe") + deadline = time.monotonic() + timeout_seconds + while True: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + return cls(descriptor) + except BlockingIOError: + if time.monotonic() >= deadline: + raise RuntimeFailure("another capture instance is active") from None + time.sleep(min(0.05, max(0.0, deadline - time.monotonic()))) + except Exception: + os.close(descriptor) + raise + + def release(self) -> None: + if self._file_descriptor is None: + return + descriptor, self._file_descriptor = self._file_descriptor, None + with suppress(OSError): + fcntl.flock(descriptor, fcntl.LOCK_UN) + with suppress(OSError): + os.close(descriptor) + + +@dataclass(slots=True) +class _Resources: + repository: CaptureRepository + corpus: CaptureCorpus + lock: InstanceLock + + def close(self) -> None: + self.lock.release() + + +class _CaptureControl(CaptureControlPort): + """Serialize one optional armed intake beneath a running local service.""" + + def __init__( + self, + runtime: CaptureRuntime, + repository: CaptureRepository, + ) -> None: + self._runtime = runtime + self._repository = repository + self._lock = RLock() + self._cancelled: Event | None = None + self._intake: IntakePort | None = None + self._worker: Thread | None = None + self._state = "disarmed" + + def status(self) -> CaptureSessionStatus: + with self._lock: + self._reap_finished_worker() + intake = self._intake + depth = self._queue_depth(intake) + return CaptureSessionStatus( + armed=self._state in {"armed", "stopping"}, + queue_depth=depth, + queue_capacity=self._runtime.config.limits.max_queue_items, + ) + + def start_session(self) -> ControlOutcome: + with self._lock: + self._reap_finished_worker() + if self._state == "armed": + return "already_armed" + if self._state == "stopping": + return "conflict" + if self._worker is not None: + return "conflict" + intake: IntakePort | None = None + try: + intake = self._runtime.intake_factory(self._runtime._watch_config()) + # Durable media always becomes eligible before the watcher is armed. + intake.reconcile() + intake.arm() + except Exception: + if intake is not None: + with suppress(Exception): + intake.shutdown() + self._state = "disarmed" + return "conflict" + assert intake is not None + cancelled = Event() + worker = Thread( + target=self._drain_loop, + args=(intake, cancelled), + name="shd-capture-drain", + daemon=True, + ) + self._intake = intake + self._cancelled = cancelled + self._worker = worker + self._state = "armed" + try: + worker.start() + except Exception: + self._state = "disarmed" + self._intake = self._cancelled = self._worker = None + with suppress(Exception): + intake.shutdown() + return "conflict" + return "started" + + def stop_session(self) -> ControlOutcome: + with self._lock: + self._reap_finished_worker() + worker = self._worker + if worker is None: + return "already_stopped" + if self._state == "stopping": + return "conflict" + self._state = "stopping" + cancelled = self._cancelled + intake = self._intake + return "stopped" if self._stop_worker(worker, intake, cancelled) else "conflict" + + def shutdown(self) -> bool: + """Request child shutdown before the service gives up its instance lock.""" + + with self._lock: + self._reap_finished_worker() + worker = self._worker + if worker is None: + return True + self._state = "stopping" + intake = self._intake + cancelled = self._cancelled + return self._stop_worker(worker, intake, cancelled) + + def defer_resource_close(self, close: Callable[[], None]) -> None: + """Release the service lock only after a timed-out drain worker exits.""" + + with self._lock: + self._reap_finished_worker() + worker = self._worker + if worker is None: + close() + return + + def wait_for_worker() -> None: + worker.join() + close() + + Thread(target=wait_for_worker, name="shd-capture-release", daemon=True).start() + + def _stop_worker( + self, + worker: Thread, + intake: IntakePort | None, + cancelled: Event | None, + ) -> bool: + if cancelled is not None: + cancelled.set() + timeout = float(self._runtime.config.lifecycle.shutdown_timeout_seconds) + initial_timeout = timeout / 2 + # Never hold the control lock while waiting for a drain or child cleanup. + worker.join(timeout=initial_timeout) + if worker.is_alive(): + # A receipt in flight remains durable because cancellation prevents its + # acknowledgement. Stopping intake unblocks watcher/helper cleanup. + if intake is not None: + with suppress(Exception): + intake.shutdown() + worker.join(timeout=timeout - initial_timeout) + with self._lock: + self._reap_finished_worker() + return not worker.is_alive() + + def _drain_loop(self, intake: IntakePort, cancelled: Event) -> None: + try: + while not cancelled.is_set(): + intake.tick() + self._runtime._drain(intake, self._repository, cancelled.is_set) + cancelled.wait(_POLL_SECONDS) + except Exception: + # The HTTP boundary exposes only bounded armed/queue state, not an + # intake, storage, helper, or media exception. + pass + finally: + with suppress(Exception): + intake.shutdown() + with self._lock: + if self._intake is intake: + self._state = "disarmed" + + def _reap_finished_worker(self) -> None: + if self._worker is not None and not self._worker.is_alive(): + self._worker = self._cancelled = self._intake = None + self._state = "disarmed" + + def _queue_depth(self, intake: IntakePort | None) -> int: + if intake is None: + return 0 + depths: list[int] = [] + for name in ("queue_depth", "helper_queue_depth"): + try: + value = getattr(intake, name, 0) + except Exception: + value = 0 + if not isinstance(value, bool) and isinstance(value, int) and value >= 0: + depths.append(value) + return min(sum(depths), self._runtime.config.limits.max_queue_items) + + +@contextmanager +def _verified_png_file(path: Path, max_bytes: int) -> Iterator[Iterable[bytes]]: + """Yield a one-descriptor PNG stream and reject unsafe input without path disclosure.""" + + candidate = path.absolute() + if len(str(candidate)) > 240: + raise RuntimeFailure("input image is not a safe PNG") + try: + descriptor = os.open(candidate, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as error: + raise RuntimeFailure("input image is not a safe PNG") from error + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or stat.S_IMODE(info.st_mode) & 0o022 + or not 8 <= info.st_size <= max_bytes + ): + raise RuntimeFailure("input image is not a safe PNG") + + def chunks() -> Iterator[bytes]: + first = os.read(descriptor, len(PNG_SIGNATURE)) + if first != PNG_SIGNATURE: + raise RuntimeFailure("input image is not a safe PNG") + total = len(first) + yield first + while chunk := os.read(descriptor, 64 * 1024): + total += len(chunk) + if total > max_bytes: + raise RuntimeFailure("input image is not a safe PNG") + yield chunk + if total != info.st_size: + raise RuntimeFailure("input image is not a safe PNG") + + yield chunks() + finally: + os.close(descriptor) + + +class CaptureRuntime: + """Own utility-local storage, then stream verified media receipts into it.""" + + def __init__( + self, + config: AppConfig, + paths: RuntimePaths, + *, + migrations_dir: Path | None = None, + intake_factory: Callable[[WatchConfig], IntakePort] = ClipboardIntake, + ) -> None: + self.config = config + self.paths = paths + self.migrations_dir = ( + Path(__file__).resolve().parents[2] / "migrations" + if migrations_dir is None + else migrations_dir + ) + self.intake_factory = intake_factory + + def _watch_config(self) -> WatchConfig: + return WatchConfig( + spool=self.paths.cache_dir / "spool", + max_image_bytes=self.config.limits.max_image_bytes, + max_queue_items=self.config.limits.max_queue_items, + policy_version=_POLICY_VERSION, + shutdown_timeout_seconds=float(min(30, self.config.lifecycle.shutdown_timeout_seconds)), + ) + + def validate_plan(self) -> None: + """Validate static configuration and paths without creating runtime resources.""" + + self._watch_config() + if not self.migrations_dir.is_absolute() or not self.migrations_dir.is_dir(): + raise RuntimeFailure("capture runtime resources are unavailable") + StorageDatabase(self.paths.data_dir, self.migrations_dir) + ContentAddressedBlobStore(self.paths.data_dir) + + def _start(self) -> _Resources: + self.validate_plan() + prepare_private_directories(self.paths) + lock = InstanceLock.acquire( + self.paths.instance_lock_file, self.config.lifecycle.lock_timeout_seconds + ) + try: + database = StorageDatabase(self.paths.data_dir, self.migrations_dir) + database.migrate() + blobs = ContentAddressedBlobStore(self.paths.data_dir) + blobs.reconcile() + JobLeases(database).recover() + corpus = CaptureCorpus( + database, + blobs, + max_media_bytes=self.config.limits.max_image_bytes, + ) + corpus.reconcile_deletions() + return _Resources(CaptureRepository(database, blobs), corpus, lock) + except RuntimeFailure: + lock.release() + raise + except Exception as error: + lock.release() + raise RuntimeFailure("capture runtime initialization failed") from error + + def ingest_image(self, image: Path) -> str: + """Persist one descriptor-verified PNG as a unique capture occurrence.""" + + resources = self._start() + token = secrets.token_hex(16) + try: + with _verified_png_file(image, self.config.limits.max_image_bytes) as stream: + return resources.repository.receive( + f"capture-{token}", + f"image-{token}", + stream, + media_type="image/png", + max_bytes=self.config.limits.max_image_bytes, + ) + except RuntimeFailure: + raise + except Exception as error: + raise RuntimeFailure("image capture could not be stored") from error + finally: + resources.close() + + def _drain( + self, + intake: IntakePort, + repository: CaptureRepository, + cancellation_requested: Callable[[], bool], + ) -> None: + while ( + not cancellation_requested() and (payload := intake.take_helper_receipt()) is not None + ): + intake.accept_helper_receipt( + payload, + occurrence_id=f"capture-{secrets.token_hex(16)}", + processing_version=_PROCESSING_VERSION, + ) + while not cancellation_requested() and (receipt := intake.take()) is not None: + with intake.open_verified(receipt) as reader: + repository.receive( + receipt.occurrence_id, + receipt.receipt_token, + reader, + media_type="image/png", + max_bytes=self.config.limits.max_image_bytes, + ) + if cancellation_requested(): + return + if not intake.acknowledge(receipt): + raise RuntimeFailure("capture receipt acknowledgement failed") + + def serve(self) -> None: + """Run one local web service with unarmed, durable capture control.""" + + bootstrap_token = new_bootstrap_token() + resources = self._start() + control = _CaptureControl(self, resources.repository) + authority = ( + f"[{self.config.server.host}]" + if ":" in self.config.server.host + else self.config.server.host + ) + try: + app = create_app( + config=WebConfig( + server=self.config.server, + max_capture_items=min(64, self.config.limits.max_queue_items), + max_image_bytes=self.config.limits.max_image_bytes, + ), + capture_control=control, + corpus=resources.corpus, + bootstrap_token=bootstrap_token, + ) + print( + f"Open local review service: http://{authority}:{self.config.server.port}/" + f"#bootstrap={bootstrap_token}" + ) + print("H1: capture is disarmed; use Start Session before taking a screenshot.") + try: + uvicorn.run( + app, + host=self.config.server.host, + port=self.config.server.port, + reload=False, + workers=1, + proxy_headers=False, + ) + except SystemExit as error: + if error.code not in (None, 0): + raise RuntimeFailure("capture service failed") from error + raise + except KeyboardInterrupt: + return + except RuntimeFailure: + raise + except Exception as error: + raise RuntimeFailure("capture service failed") from error + finally: + if control.shutdown(): + resources.close() + else: + control.defer_resource_close(resources.close) + raise RuntimeFailure("capture service shutdown failed") + + def watch(self, *, cancelled: Event | None = None, poll_seconds: float = _POLL_SECONDS) -> None: + """Run the armed intake until cancellation, signal, or keyboard interruption.""" + + if not 0 < poll_seconds <= 1: + raise ValueError("invalid runtime poll interval") + resources = self._start() + intake: IntakePort | None = None + stop_requested = Event() + handlers: dict[signal.Signals, object] = {} + + def request_stop(_signal_number: int, _frame: object) -> None: + stop_requested.set() + + try: + if current_thread() is main_thread(): + for configured_signal in (signal.SIGINT, signal.SIGTERM): + handlers[configured_signal] = signal.signal(configured_signal, request_stop) + intake = self.intake_factory(self._watch_config()) + intake.reconcile() + intake.arm() + while not stop_requested.is_set() and (cancelled is None or not cancelled.is_set()): + intake.tick() + self._drain( + intake, + resources.repository, + lambda: ( + stop_requested.is_set() or (cancelled is not None and cancelled.is_set()) + ), + ) + time.sleep(poll_seconds) + except KeyboardInterrupt: + return + except RuntimeFailure: + raise + except Exception as error: + raise RuntimeFailure("capture watch failed") from error + finally: + if intake is not None: + with suppress(OSError, RuntimeError): + intake.shutdown() + for restored_signal, handler in handlers.items(): + signal.signal(restored_signal, cast(signal.Handlers, handler)) + resources.close() diff --git a/tools/game-data-capture/src/game_data_capture/storage/__init__.py b/tools/game-data-capture/src/game_data_capture/storage/__init__.py new file mode 100644 index 0000000..a2481f3 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/storage/__init__.py @@ -0,0 +1,15 @@ +"""Utility-local durable storage; it never opens the SHD.build application database.""" + +from game_data_capture.storage.blobs import BlobCorruptionError, ContentAddressedBlobStore +from game_data_capture.storage.corpus import CaptureCorpus +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import CaptureRepository, StorageError + +__all__ = [ + "BlobCorruptionError", + "CaptureCorpus", + "CaptureRepository", + "ContentAddressedBlobStore", + "StorageDatabase", + "StorageError", +] diff --git a/tools/game-data-capture/src/game_data_capture/storage/blobs.py b/tools/game-data-capture/src/game_data_capture/storage/blobs.py new file mode 100644 index 0000000..097ad29 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/storage/blobs.py @@ -0,0 +1,264 @@ +"""Verified immutable blobs with no symlink traversal at storage boundaries.""" + +from __future__ import annotations + +import hashlib +import os +import secrets +import time +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from pathlib import Path +from stat import S_IMODE, S_ISDIR, S_ISREG + + +class BlobCorruptionError(RuntimeError): + pass + + +class BlobLimitError(ValueError): + pass + + +class BlobLockError(RuntimeError): + pass + + +def _digest(value: str) -> None: + if len(value) != 64 or any(char not in "0123456789abcdef" for char in value): + raise ValueError("invalid SHA-256 digest") + + +class ContentAddressedBlobStore: + def __init__(self, root: Path) -> None: + if not root.is_absolute(): + raise ValueError("blob root must be absolute") + self.root = root + self.blobs = root / "blobs" + self.temporary = root / "tmp" + self.lock_path = root / ".blob-operation.lock" + + @contextmanager + def operation_lock(self, *, timeout_seconds: float = 1.0) -> Iterator[None]: + """Hold the private cross-process lock for receipt finalization or deletion.""" + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, int | float) + or not 0 < timeout_seconds <= 5 + ): + raise ValueError("blob lock timeout is invalid") + self._directory(self.root, create=True) + try: + fd = os.open(self.lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + except OSError as error: + raise BlobLockError("blob operation lock is unavailable") from error + try: + info = os.fstat(fd) + if not S_ISREG(info.st_mode) or S_IMODE(info.st_mode) != 0o600: + raise BlobLockError("blob operation lock is unsafe") + import fcntl + + deadline = time.monotonic() + timeout_seconds + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise BlobLockError("blob operation lock is unavailable") from None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BlobLockError("blob operation lock is unavailable") from None + time.sleep(min(0.02, remaining)) + yield + finally: + try: + import fcntl + + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + def _directory(self, path: Path, *, create: bool) -> None: + if path.is_symlink(): + raise BlobCorruptionError("storage directory is unsafe") + if create: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + if not path.exists() or path.is_symlink() or not S_ISDIR(path.lstat().st_mode): + raise BlobCorruptionError("storage directory is unsafe") + path.chmod(0o700) + + def _path(self, digest: str) -> Path: + _digest(digest) + return self.blobs / digest[:2] / digest[2:] + + def _fsync_directory(self, path: Path) -> None: + if path.is_symlink(): + raise BlobCorruptionError("storage directory is unsafe") + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + def _verify_fd(self, fd: int, digest: str, expected_size: int | None) -> int: + stat = os.fstat(fd) + if not S_ISREG(stat.st_mode): + raise BlobCorruptionError("blob is not a regular file") + if expected_size is not None and stat.st_size != expected_size: + raise BlobCorruptionError("blob content does not match receipt") + os.lseek(fd, 0, os.SEEK_SET) + hasher = hashlib.sha256() + size = 0 + while chunk := os.read(fd, 1024 * 1024): + size += len(chunk) + hasher.update(chunk) + if hasher.hexdigest() != digest or (expected_size is not None and size != expected_size): + raise BlobCorruptionError("blob content does not match receipt") + os.lseek(fd, 0, os.SEEK_SET) + return size + + def put( + self, chunks: Iterable[bytes], *, max_bytes: int, expected_sha256: str | None = None + ) -> tuple[str, int]: + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise BlobLimitError("blob limit must be positive") + if expected_sha256 is not None: + _digest(expected_sha256) + self._directory(self.root, create=True) + self._directory(self.blobs, create=True) + self._directory(self.temporary, create=True) + temporary = self.temporary / f".{secrets.token_hex(16)}.partial" + hasher = hashlib.sha256() + size = 0 + try: + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + for chunk in chunks: + if not isinstance(chunk, bytes) or not chunk: + raise BlobLimitError("blob chunks must be non-empty bytes") + size += len(chunk) + if size > max_bytes: + raise BlobLimitError("blob exceeds configured size limit") + hasher.update(chunk) + view = memoryview(chunk) + while view: + written = os.write(fd, view) + if written <= 0: + raise OSError("short blob write") + view = view[written:] + os.fsync(fd) + finally: + os.close(fd) + digest = hasher.hexdigest() + if size == 0 or (expected_sha256 is not None and digest != expected_sha256): + raise BlobCorruptionError("blob digest verification failed") + with self.operation_lock(): + destination = self._path(digest) + new_prefix = not destination.parent.exists() + self._directory(destination.parent, create=True) + if new_prefix: + self._fsync_directory(self.blobs) + directory_fd = os.open(destination.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + if destination.exists(): + self.verify(digest, expected_size=size) + else: + os.replace(temporary, destination) + os.chmod(destination, 0o600) + os.fsync(directory_fd) + finally: + os.close(directory_fd) + self.verify(digest, expected_size=size) + return digest, size + finally: + if temporary.exists() and not temporary.is_symlink(): + temporary.unlink() + + def _open_verified(self, digest: str, expected_size: int | None = None) -> tuple[int, int]: + path = self._path(digest) + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except OSError as error: + raise BlobCorruptionError("blob is missing or unsafe") from error + try: + return fd, self._verify_fd(fd, digest, expected_size) + except Exception: + os.close(fd) + raise + + def verify(self, digest: str, *, expected_size: int | None = None) -> int: + fd, size = self._open_verified(digest, expected_size) + os.close(fd) + return size + + def read( + self, digest: str, *, max_bytes: int, expected_size: int | None = None + ) -> Iterator[bytes]: + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise BlobLimitError("blob limit must be positive") + fd, size = self._open_verified(digest, expected_size) + if size > max_bytes: + os.close(fd) + raise BlobLimitError("blob exceeds read limit") + try: + while chunk := os.read(fd, 1024 * 1024): + yield chunk + finally: + os.close(fd) + + def delete_verified(self, digest: str, *, expected_size: int) -> bool: + """Unlink one verified immutable blob without following a replacement path.""" + if ( + isinstance(expected_size, bool) + or not isinstance(expected_size, int) + or expected_size <= 0 + ): + raise BlobLimitError("blob size is invalid") + fd, _ = self._open_verified(digest, expected_size) + path = self._path(digest) + try: + parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + current = os.stat(path.name, dir_fd=parent_fd, follow_symlinks=False) + opened = os.fstat(fd) + if (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino): + raise BlobCorruptionError("blob changed during deletion") + os.unlink(path.name, dir_fd=parent_fd) + os.fsync(parent_fd) + return True + finally: + os.close(parent_fd) + except OSError as error: + raise BlobCorruptionError("blob is missing or unsafe") from error + finally: + os.close(fd) + + def reconcile(self) -> list[str]: + if not self.temporary.exists() or self.temporary.is_symlink(): + return [] + self._directory(self.temporary, create=False) + removed = [] + for entry in self.temporary.iterdir(): + if not entry.is_symlink() and entry.is_file() and entry.name.endswith(".partial"): + entry.unlink() + removed.append(entry.name) + return removed + + def purge_dry_run(self, digests: set[str]) -> list[Path]: + if not self.blobs.exists() or self.blobs.is_symlink(): + return [] + self._directory(self.blobs, create=False) + candidates = [] + for prefix in self.blobs.iterdir(): + if prefix.is_symlink() or not prefix.is_dir(): + continue + for entry in prefix.iterdir(): + if ( + not entry.is_symlink() + and entry.is_file() + and prefix.name + entry.name not in digests + ): + candidates.append(entry.relative_to(self.root)) + return candidates diff --git a/tools/game-data-capture/src/game_data_capture/storage/corpus.py b/tools/game-data-capture/src/game_data_capture/storage/corpus.py new file mode 100644 index 0000000..3036a16 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/storage/corpus.py @@ -0,0 +1,499 @@ +"""Durable, safe projections for the reviewed local capture corpus port.""" + +from __future__ import annotations + +import json +import re +import sqlite3 +from collections.abc import Mapping, Sequence + +from game_data_capture.domain.validation import ( + ContractValidationError, + validate_classification, +) +from game_data_capture.storage.blobs import ( + BlobCorruptionError, + BlobLockError, + ContentAddressedBlobStore, +) +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import StorageError, _now +from game_data_capture.web.app import CaptureRecord, MediaDescriptor, MutationOutcome + +_CAPTURE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$") +_MEDIA_TYPES = frozenset({"image/png", "image/webp", "image/avif"}) +_CLASSIFICATION_KEYS = frozenset( + {"family", "view", "gear_slot", "record_types", "confidence", "abstention_reason"} +) +_MAX_MEDIA_BYTES = 64 * 1024 * 1024 + + +def _capture_id(value: str) -> bool: + return isinstance(value, str) and bool(_CAPTURE_ID.fullmatch(value)) + + +def _classification(value: Mapping[str, object]) -> str: + if not isinstance(value, Mapping) or set(value) != _CLASSIFICATION_KEYS: + raise ValueError("classification does not match the frozen taxonomy shape") + payload = dict(value) + try: + validate_classification(payload) + confidence = payload["confidence"] + record_types = payload["record_types"] + if isinstance(confidence, bool) or not isinstance(confidence, int | float): + raise ValueError("classification confidence is invalid") + if not isinstance(record_types, list) or any( + not isinstance(record_type, str) for record_type in record_types + ): + raise ValueError("classification record types are invalid") + canonical_confidence = 0.0 if confidence == 0 else float(confidence) + canonical = { + "family": payload["family"], + "view": payload["view"], + "gear_slot": payload["gear_slot"], + "record_types": sorted(record_types), + "confidence": canonical_confidence, + "abstention_reason": payload["abstention_reason"], + } + encoded = json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + except (ContractValidationError, TypeError, ValueError) as error: + raise ValueError("classification does not match the frozen taxonomy shape") from error + if not 2 <= len(encoded) <= 4096: + raise ValueError("classification does not match the frozen taxonomy shape") + return encoded + + +def _stored_classification(value: object) -> Mapping[str, object]: + if not isinstance(value, str) or not 2 <= len(value) <= 4096: + raise StorageError("stored corpus record is invalid") + try: + decoded = json.loads(value) + except (TypeError, ValueError) as error: + raise StorageError("stored corpus record is invalid") from error + if not isinstance(decoded, dict) or set(decoded) != _CLASSIFICATION_KEYS: + raise StorageError("stored corpus record is invalid") + try: + validate_classification(decoded) + except ContractValidationError as error: + raise StorageError("stored corpus record is invalid") from error + return decoded + + +class CaptureCorpus: + """SQLite-backed implementation of the reviewed ``CaptureCorpusPort``. + + The corpus exposes only opaque capture identifiers. It serializes writes with + ``BEGIN IMMEDIATE`` so repeated labels and rejections have deterministic + idempotence, while differing labels preserve the replaced value as evidence. + """ + + def __init__( + self, + database: StorageDatabase, + blobs: ContentAddressedBlobStore, + *, + max_media_bytes: int = _MAX_MEDIA_BYTES, + ) -> None: + if ( + isinstance(max_media_bytes, bool) + or not isinstance(max_media_bytes, int) + or not 1 <= max_media_bytes <= _MAX_MEDIA_BYTES + ): + raise ValueError("corpus media limit is invalid") + self.database = database + self.blobs = blobs + self.max_media_bytes = max_media_bytes + + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 256: + raise ValueError("capture list limit is invalid") + with self.database.connect() as connection: + rows = connection.execute( + "SELECT c.capture_id, c.state, c.received_at, l.classification_json " + "FROM captures AS c LEFT JOIN capture_labels AS l USING(capture_id) " + "ORDER BY c.received_at DESC, c.capture_id DESC LIMIT ?", + (limit,), + ) + return [self._record(row) for row in rows] + + def get_capture(self, capture_id: str) -> CaptureRecord | None: + if not _capture_id(capture_id): + return None + with self.database.connect() as connection: + row = connection.execute( + "SELECT c.capture_id, c.state, c.received_at, l.classification_json " + "FROM captures AS c LEFT JOIN capture_labels AS l USING(capture_id) " + "WHERE c.capture_id = ?", + (capture_id,), + ).fetchone() + return None if row is None else self._record(row) + + def open_media(self, capture_id: str) -> MediaDescriptor | None: + if not _capture_id(capture_id): + return None + with self.database.connect() as connection: + row = connection.execute( + "SELECT b.sha256, b.byte_size, b.media_type FROM captures AS c " + "JOIN blobs AS b ON b.sha256 = c.blob_sha256 WHERE c.capture_id = ?", + (capture_id,), + ).fetchone() + if row is None: + return None + digest, size, media_type = row["sha256"], row["byte_size"], row["media_type"] + if ( + not isinstance(digest, str) + or isinstance(size, bool) + or not isinstance(size, int) + or not 1 <= size <= self.max_media_bytes + or not isinstance(media_type, str) + or media_type not in _MEDIA_TYPES + ): + raise StorageError("stored corpus media is invalid") + try: + chunks = self.blobs.read( + digest, + max_bytes=self.max_media_bytes, + expected_size=size, + ) + except (BlobCorruptionError, ValueError) as error: + raise StorageError("stored corpus media is unavailable") from error + return MediaDescriptor(media_type=media_type, byte_size=size, chunks=chunks) + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: + if not _capture_id(capture_id): + return "not_found" + encoded = _classification(classification) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + if not self._exists(connection, capture_id): + connection.execute("COMMIT") + return "not_found" + if self._has_rejection(connection, capture_id): + connection.execute("COMMIT") + return "conflict" + current = connection.execute( + "SELECT classification_json, revision FROM capture_labels WHERE capture_id = ?", + (capture_id,), + ).fetchone() + if current is None: + connection.execute( + "INSERT INTO capture_labels(capture_id, classification_json, revision, " + "labeled_at) VALUES (?, ?, 0, ?)", + (capture_id, encoded, _now()), + ) + else: + current_json = current["classification_json"] + _stored_classification(current_json) + if current_json == encoded: + connection.execute("COMMIT") + return "updated" + revision = current["revision"] + revision_is_valid = ( + not isinstance(revision, bool) + and isinstance(revision, int) + and revision >= 0 + ) + if not revision_is_valid: + raise StorageError("stored corpus label is invalid") + assert isinstance(revision, int) and not isinstance(revision, bool) + connection.execute( + "INSERT INTO capture_label_evidence(capture_id, replaced_revision, " + "classification_json, replaced_at) VALUES (?, ?, ?, ?)", + (capture_id, revision, current_json, _now()), + ) + result = connection.execute( + "UPDATE capture_labels SET classification_json = ?, " + "revision = revision + 1, labeled_at = ? " + "WHERE capture_id = ? AND revision = ?", + (encoded, _now(), capture_id, revision), + ) + if result.rowcount != 1: + connection.execute("ROLLBACK") + return "conflict" + connection.execute("COMMIT") + return "updated" + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: + if not _capture_id(capture_id): + return "not_found" + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + if not self._exists(connection, capture_id): + connection.execute("COMMIT") + return "not_found" + if not self._has_rejection(connection, capture_id): + connection.execute( + "INSERT INTO capture_rejections(capture_id, rejected_at) VALUES (?, ?)", + (capture_id, _now()), + ) + connection.execute("COMMIT") + return "updated" + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: + if not _capture_id(capture_id): + return "not_found" + try: + with self.blobs.operation_lock(): + return self._delete_locked(capture_id) + except BlobLockError as error: + raise StorageError("blob operation is unavailable") from error + + def _delete_locked(self, capture_id: str) -> MutationOutcome: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT blob_sha256, state, received_at, receipt_token, " + "failure_reason, revision " + "FROM captures WHERE capture_id = ?", + (capture_id,), + ).fetchone() + if row is None: + connection.execute("COMMIT") + return "not_found" + if self._retained(connection, capture_id) or not self._deletable_capture(row): + connection.execute("COMMIT") + return "retained" + digest = row["blob_sha256"] + blob = connection.execute( + "SELECT byte_size FROM blobs WHERE sha256 = ?", (digest,) + ).fetchone() + if ( + not isinstance(digest, str) + or blob is None + or isinstance(blob["byte_size"], bool) + or not isinstance(blob["byte_size"], int) + or not 1 <= blob["byte_size"] <= self.max_media_bytes + ): + connection.execute("COMMIT") + return "retained" + try: + self.blobs.verify(digest, expected_size=blob["byte_size"]) + except (BlobCorruptionError, ValueError): + connection.execute("COMMIT") + return "retained" + references = connection.execute( + "SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest,) + ).fetchone()[0] + if not isinstance(references, int) or references < 1: + connection.execute("ROLLBACK") + return "conflict" + if references == 1: + connection.execute( + "INSERT INTO capture_deletion_intents(blob_sha256, capture_id, byte_size, " + "capture_state, received_at, receipt_token, failure_reason, revision, " + "created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + digest, + capture_id, + blob["byte_size"], + row["state"], + row["received_at"], + row["receipt_token"], + row["failure_reason"], + row["revision"], + _now(), + ), + ) + result = connection.execute( + "DELETE FROM captures WHERE capture_id = ?", (capture_id,) + ) + if result.rowcount != 1: + connection.execute("ROLLBACK") + return "conflict" + connection.execute("COMMIT") + if references > 1: + return "updated" + return ( + "updated" + if self._reconcile_locked(limit=1, restore_on_failure=True) + else "retained" + ) + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + + @staticmethod + def _exists(connection: sqlite3.Connection, capture_id: str) -> bool: + return ( + connection.execute( + "SELECT 1 FROM captures WHERE capture_id = ?", (capture_id,) + ).fetchone() + is not None + ) + + @staticmethod + def _has_rejection(connection: sqlite3.Connection, capture_id: str) -> bool: + return ( + connection.execute( + "SELECT 1 FROM capture_rejections WHERE capture_id = ?", (capture_id,) + ).fetchone() + is not None + ) + + @staticmethod + def _retained(connection: sqlite3.Connection, capture_id: str) -> bool: + tables = ( + "jobs", + "ocr_runs", + "capture_labels", + "capture_label_evidence", + "capture_rejections", + ) + if any( + connection.execute( + f"SELECT 1 FROM {table} WHERE capture_id = ? LIMIT 1", (capture_id,) + ).fetchone() + is not None + for table in tables + ): + return True + if ( + connection.execute( + "SELECT 1 FROM runtime_failures WHERE subject_type = 'capture' " + "AND subject_id = ? LIMIT 1", + (capture_id,), + ).fetchone() + is not None + ): + return True + return ( + connection.execute( + "SELECT 1 FROM lineage WHERE child_id = ? OR parent_id = ? LIMIT 1", + (capture_id, capture_id), + ).fetchone() + is not None + ) + + @staticmethod + def _deletable_capture(row: sqlite3.Row) -> bool: + return row["state"] in {"stored", "deduplicated"} and row["failure_reason"] is None + + def reconcile_deletions(self, *, limit: int = 64) -> int: + """Finish bounded committed deletion intents after a crash or unlink failure.""" + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 64: + raise ValueError("deletion reconciliation limit is invalid") + try: + with self.blobs.operation_lock(): + return self._reconcile_locked(limit=limit, restore_on_failure=False) + except BlobLockError as error: + raise StorageError("blob operation is unavailable") from error + + def _reconcile_locked(self, *, limit: int, restore_on_failure: bool) -> int: + with self.database.connect() as connection: + intents = list( + connection.execute( + "SELECT blob_sha256, byte_size, capture_id, capture_state, received_at, " + "receipt_token, failure_reason, revision FROM capture_deletion_intents " + "ORDER BY created_at, capture_id LIMIT ?", + (limit,), + ) + ) + completed = 0 + for intent in intents: + digest, size = intent["blob_sha256"], intent["byte_size"] + if not isinstance(digest, str) or isinstance(size, bool) or not isinstance(size, int): + continue + try: + self.blobs.delete_verified(digest, expected_size=size) + except BlobCorruptionError: + try: + self.blobs._path(digest).lstat() + except FileNotFoundError: + pass + else: + if restore_on_failure and self._restore_intent(intent): + continue + if restore_on_failure: + raise StorageError("blob deletion could not be completed") from None + continue + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + active = connection.execute( + "SELECT 1 FROM captures WHERE blob_sha256 = ?", (digest,) + ).fetchone() + if active is not None: + connection.execute("ROLLBACK") + continue + result = connection.execute( + "DELETE FROM capture_deletion_intents WHERE blob_sha256 = ?", (digest,) + ) + connection.execute("DELETE FROM blobs WHERE sha256 = ?", (digest,)) + connection.execute("COMMIT") + completed += result.rowcount + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + return completed + + def _restore_intent(self, intent: sqlite3.Row) -> bool: + digest = intent["blob_sha256"] + size = intent["byte_size"] + if not isinstance(digest, str) or isinstance(size, bool) or not isinstance(size, int): + return False + try: + self.blobs.verify(digest, expected_size=size) + except (BlobCorruptionError, ValueError): + return False + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + "INSERT INTO captures(capture_id, blob_sha256, state, received_at, " + "receipt_token, failure_reason, revision) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + intent["capture_id"], + digest, + intent["capture_state"], + intent["received_at"], + intent["receipt_token"], + intent["failure_reason"], + intent["revision"], + ), + ) + result = connection.execute( + "DELETE FROM capture_deletion_intents WHERE blob_sha256 = ?", (digest,) + ) + connection.execute("COMMIT") + return result.rowcount == 1 + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + + @staticmethod + def _record(row: sqlite3.Row) -> CaptureRecord: + capture_id, state, received_at = ( + row["capture_id"], + row["state"], + row["received_at"], + ) + if ( + not _capture_id(capture_id) + or not isinstance(state, str) + or not isinstance(received_at, str) + or not 1 <= len(received_at) <= 40 + or "\r" in received_at + or "\n" in received_at + ): + raise StorageError("stored corpus record is invalid") + raw_label = row["classification_json"] + return CaptureRecord( + capture_id=capture_id, + state=state, + received_at=received_at, + classification=None if raw_label is None else _stored_classification(raw_label), + ) diff --git a/tools/game-data-capture/src/game_data_capture/storage/database.py b/tools/game-data-capture/src/game_data_capture/storage/database.py new file mode 100644 index 0000000..d6a56b6 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/storage/database.py @@ -0,0 +1,124 @@ +"""Private SQLite setup and immutable ordered utility migrations.""" + +from __future__ import annotations + +import hashlib +import os +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path +from stat import S_ISDIR, S_ISLNK + + +class MigrationError(RuntimeError): + """The local migration history cannot safely be applied.""" + + +def _private_directory(path: Path) -> None: + if path.is_symlink(): + raise MigrationError("utility root must not be a symlink") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + mode = path.lstat().st_mode + if S_ISLNK(mode) or not S_ISDIR(mode): + raise MigrationError("utility root must be a directory") + path.chmod(0o700) + + +class StorageDatabase: + def __init__(self, root: Path, migrations_dir: Path) -> None: + if not root.is_absolute() or not migrations_dir.is_absolute(): + raise ValueError("storage paths must be absolute") + self.root = root + self.migrations_dir = migrations_dir + self.path = root / "capture.sqlite3" + + def connect(self) -> sqlite3.Connection: + _private_directory(self.root) + if self.path.is_symlink(): + raise MigrationError("database file must not be a symlink") + connection: sqlite3.Connection | None = None + try: + connection = sqlite3.connect(self.path, isolation_level=None, timeout=5.0) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + journal = connection.execute("PRAGMA journal_mode = WAL").fetchone()[0] + connection.execute("PRAGMA busy_timeout = 5000") + connection.execute("PRAGMA synchronous = FULL") + checks = ( + connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1, + journal == "wal", + connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000, + connection.execute("PRAGMA synchronous").fetchone()[0] == 2, + ) + if not all(checks): + raise MigrationError("SQLite connection invariants were not enabled") + for candidate in ( + self.path, + self.path.with_name(self.path.name + "-wal"), + self.path.with_name(self.path.name + "-shm"), + ): + if candidate.exists() and not candidate.is_symlink(): + candidate.chmod(0o600) + return connection + except Exception: + if connection is not None: + connection.close() + raise + + def _migrations(self) -> list[tuple[int, Path, str]]: + found: list[tuple[int, Path, str]] = [] + versions: set[int] = set() + for path in self.migrations_dir.glob("*.sql"): + if len(path.name) < 5 or not path.name[:3].isdigit() or path.name[3] != "_": + raise MigrationError("migration filename is invalid") + version = int(path.name[:3]) + if version in versions: + raise MigrationError("migration versions must be unique") + versions.add(version) + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + found.append((version, path, checksum)) + found.sort(key=lambda item: item[0]) + if not found: + raise MigrationError("no storage migrations found") + return found + + def migrate(self) -> None: + migrations = self._migrations() + with self.connect() as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version INTEGER PRIMARY KEY, filename TEXT NOT NULL UNIQUE, " + "checksum TEXT NOT NULL, applied_at TEXT NOT NULL)" + ) + applied = list( + connection.execute( + "SELECT version, filename, checksum FROM schema_migrations ORDER BY version" + ) + ) + if len(applied) > len(migrations): + raise MigrationError("applied migration is no longer present") + expected_prefix = migrations[: len(applied)] + for row, expected in zip(applied, expected_prefix, strict=True): + version, path, checksum = expected + if (row["version"], row["filename"], row["checksum"]) != ( + version, + path.name, + checksum, + ): + raise MigrationError("migration history is not an exact ordered prefix") + for version, path, checksum in migrations[len(applied) :]: + sql = path.read_text(encoding="utf-8") + try: + applied_at = datetime.now(UTC).isoformat().replace("'", "''") + connection.executescript( + "BEGIN IMMEDIATE;\n" + + sql + + "\nINSERT INTO schema_migrations VALUES " + + f"({version}, '{path.name}', '{checksum}', '{applied_at}');" + + "\nCOMMIT;" + ) + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + os.chmod(self.path, 0o600) diff --git a/tools/game-data-capture/src/game_data_capture/storage/repository.py b/tools/game-data-capture/src/game_data_capture/storage/repository.py new file mode 100644 index 0000000..18a3024 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/storage/repository.py @@ -0,0 +1,266 @@ +"""Utility-local capture receipts and bounded insert-only provenance.""" + +from __future__ import annotations + +import json +import re +import sqlite3 +from collections.abc import Iterable +from datetime import UTC, datetime + +from game_data_capture.domain.state_machines import require_transition +from game_data_capture.storage.blobs import ( + BlobCorruptionError, + BlobLockError, + ContentAddressedBlobStore, +) +from game_data_capture.storage.database import StorageDatabase + +_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_MEDIA = re.compile(r"^(?:image/(?:png|webp|avif)|video/(?:mp4|webm|x-matroska))$") +_SECRET = re.compile( + r"(?i)(?:sk-[a-z0-9_-]{8,}|authorization\s*[:=]\s*bearer\s+\S+|" + r"(?:token|secret|password|api[_-]?key)\s*[:=]\s*\S+|bearer\s+\S+|/[^\s]+)" +) + + +class StorageError(RuntimeError): + pass + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +def _id(value: str) -> str: + if not isinstance(value, str) or not _ID.fullmatch(value): + raise ValueError("invalid local identifier") + return value + + +def _json(value: object, maximum: int) -> str: + encoded = json.dumps(value, separators=(",", ":"), ensure_ascii=False) + if len(encoded) > maximum: + raise ValueError("provenance payload exceeds its bound") + return encoded + + +def sanitize_failure(code: str, message: str) -> tuple[str, str]: + safe_code = re.sub(r"[^a-z0-9_]", "_", code.lower())[:64] or "storage_failure" + compact = " ".join(message.split()) + compact = _SECRET.sub("[REDACTED]", compact) + if len(compact) > 512 or (len(compact) > 160 and " " not in compact[:160]): + compact = "operation failed [payload redacted]" + return safe_code, compact[:512] or "operation failed" + + +class CaptureRepository: + def __init__(self, database: StorageDatabase, blobs: ContentAddressedBlobStore) -> None: + self.database = database + self.blobs = blobs + + def receive( + self, + capture_id: str, + receipt_token: str, + chunks: Iterable[bytes], + *, + media_type: str, + max_bytes: int, + ) -> str: + _id(capture_id) + _id(receipt_token) + if not _MEDIA.fullmatch(media_type): + raise ValueError("unsupported media type") + try: + with self.blobs.operation_lock(), self.database.connect() as connection: + pending = connection.execute( + "SELECT 1 FROM capture_deletion_intents WHERE receipt_token = ?", + (receipt_token,), + ).fetchone() + if pending is not None: + raise StorageError("blob deletion is pending") + existing = connection.execute( + "SELECT capture_id FROM captures WHERE receipt_token = ?", (receipt_token,) + ).fetchone() + if existing is not None: + return str(existing["capture_id"]) + except BlobLockError as error: + raise StorageError("blob operation is unavailable") from error + try: + digest, size = self.blobs.put(chunks, max_bytes=max_bytes) + except BlobLockError as error: + raise StorageError("blob operation is unavailable") from error + try: + with self.blobs.operation_lock(): + self.blobs.verify(digest, expected_size=size) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + pending = connection.execute( + "SELECT 1 FROM capture_deletion_intents WHERE blob_sha256 = ?", + (digest,), + ).fetchone() + if pending is not None: + raise StorageError("blob deletion is pending") + existing = connection.execute( + "SELECT capture_id FROM captures WHERE receipt_token = ?", + (receipt_token,), + ).fetchone() + if existing is not None: + connection.execute("COMMIT") + return str(existing["capture_id"]) + connection.execute( + "INSERT OR IGNORE INTO blobs(sha256, byte_size, media_type, " + "created_at) " + "VALUES (?, ?, ?, ?)", + (digest, size, media_type, _now()), + ) + duplicate = connection.execute( + "SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest,) + ).fetchone()[0] + connection.execute( + "INSERT INTO captures(capture_id, blob_sha256, state, received_at, " + "receipt_token) VALUES (?, ?, ?, ?, ?)", + ( + capture_id, + digest, + "deduplicated" if duplicate else "stored", + _now(), + receipt_token, + ), + ) + connection.execute("COMMIT") + return capture_id + except Exception: + if connection.in_transaction: + connection.execute("ROLLBACK") + raise + except BlobLockError as error: + raise StorageError("blob operation is unavailable") from error + + def transition(self, capture_id: str, target: str, expected_revision: int) -> int: + _id(capture_id) + with self.database.connect() as connection: + row = connection.execute( + "SELECT state FROM captures WHERE capture_id = ? AND revision = ?", + (capture_id, expected_revision), + ).fetchone() + if row is None: + raise StorageError("stale or missing capture") + require_transition("capture", str(row["state"]), target) + result = connection.execute( + "UPDATE captures SET state = ?, revision = revision + 1 " + "WHERE capture_id = ? AND revision = ?", + (target, capture_id, expected_revision), + ) + if result.rowcount != 1: + raise StorageError("stale capture transition") + return expected_revision + 1 + + def append_ocr_run( + self, record_id: str, capture_id: str, engine_id: str, engine_version: str, payload: object + ) -> None: + if not 0 < len(engine_id) <= 100 or not 0 < len(engine_version) <= 100: + raise ValueError("OCR engine identity is invalid") + self._insert( + "ocr_runs", + ( + record_id, + capture_id, + engine_id, + engine_version, + _now(), + "completed", + _json(payload, 1_000_000), + ), + ) + + def append_region(self, record_id: str, run_id: str, payload: object) -> None: + self._insert("regions", (record_id, run_id, _json(payload, 10_000), _now())) + + def append_classification(self, record_id: str, run_id: str, payload: object) -> None: + self._insert("classifications", (record_id, run_id, _json(payload, 10_000), _now())) + + def append_candidate(self, record_id: str, run_id: str, payload: object) -> None: + self._insert( + "extraction_candidates", (record_id, run_id, _json(payload, 1_000_000), _now()) + ) + + def append_routing(self, record_id: str, candidate_id: str, payload: object) -> None: + self._insert("routing_decisions", (record_id, candidate_id, _json(payload, 10_000), _now())) + + def append_review(self, record_id: str, candidate_id: str, payload: object) -> None: + self._insert("review_events", (record_id, candidate_id, _json(payload, 10_000), _now())) + + def append_lineage(self, record_id: str, child_id: str, parent_id: str, relation: str) -> None: + if not 0 < len(relation) <= 100: + raise ValueError("lineage relation is invalid") + self._insert("lineage", (record_id, child_id, parent_id, relation, _now())) + + def append_export(self, record_id: str, state: str, manifest: object) -> None: + if not state or len(state) > 100: + raise ValueError("export state is invalid") + self._insert("exports", (record_id, state, _json(manifest, 1_000_000), _now())) + + def _insert(self, table: str, values: tuple[object, ...]) -> None: + specs = { + "ocr_runs": ( + "run_id,capture_id,engine_id,engine_version,created_at,state,payload_json", + (0, 1), + ), + "regions": ("region_id,run_id,region_json,created_at", (0, 1)), + "classifications": ("classification_id,run_id,classification_json,created_at", (0, 1)), + "extraction_candidates": ("candidate_id,run_id,candidate_json,created_at", (0, 1)), + "routing_decisions": ("decision_id,candidate_id,decision_json,created_at", (0, 1)), + "review_events": ("event_id,candidate_id,event_json,created_at", (0, 1)), + "lineage": ("lineage_id,child_id,parent_id,relation,created_at", (0, 1, 2)), + "exports": ("export_id,state,manifest_json,created_at", (0,)), + } + columns, positions = specs[table] + if len(values) != len(columns.split(",")): + raise ValueError("provenance record arity is invalid") + for position in positions: + _id(str(values[position])) + placeholders = ", ".join("?" for _ in values) + with self.database.connect() as connection: + connection.execute(f"INSERT INTO {table} ({columns}) VALUES ({placeholders})", values) + + def report(self) -> dict[str, int]: + tables = ("blobs", "captures", "jobs", "ocr_runs", "runtime_failures") + with self.database.connect() as connection: + return { + table: int(connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) + for table in tables + } + + def verify(self) -> list[str]: + with self.database.connect() as connection: + return [ + str(row["sha256"]) + for row in connection.execute("SELECT sha256, byte_size FROM blobs") + if self._invalid_blob(row) + ] + + def _invalid_blob(self, row: sqlite3.Row) -> bool: + try: + digest = row["sha256"] + size = row["byte_size"] + if not isinstance(digest, str) or isinstance(size, bool) or not isinstance(size, int): + return True + self.blobs.verify(digest, expected_size=size) + except BlobCorruptionError: + return True + return False + + def record_failure( + self, failure_id: str, subject_type: str, subject_id: str, code: str, message: str + ) -> None: + if not re.fullmatch(r"[a-z_]{1,64}", subject_type): + raise ValueError("failure subject type is invalid") + code, message = sanitize_failure(code, message) + with self.database.connect() as connection: + connection.execute( + "INSERT INTO runtime_failures VALUES (?, ?, ?, ?, ?, ?)", + (_id(failure_id), subject_type, _id(subject_id), code, message, _now()), + ) diff --git a/tools/game-data-capture/src/game_data_capture/web/app.py b/tools/game-data-capture/src/game_data_capture/web/app.py new file mode 100644 index 0000000..0bcfdb0 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/web/app.py @@ -0,0 +1,567 @@ +"""FastAPI factory for the local capture-and-labeling boundary. + +Construction only retains injected ports and process-local session state. It does +not bind a listener, initialize runtime paths, open SQLite, arm capture, read a +clipboard, or create a child process. +""" + +from __future__ import annotations + +import json +import re +import time +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Protocol, TypeVar + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from fastapi.templating import Jinja2Templates +from starlette.staticfiles import StaticFiles + +from game_data_capture.config import ServerConfig +from game_data_capture.domain.state_machines import CaptureState +from game_data_capture.domain.validation import ContractValidationError, validate_classification +from game_data_capture.web.security import ( + AuthenticatedSession, + AuthenticationError, + BodyLimitExceeded, + ClientDisconnected, + LocalAuthority, + LocalRequestLimitsMiddleware, + RequestLimits, + SecurityError, + SessionStore, + response_security_headers, + set_session_cookie, + single_header, + validate_origin, +) + +_CAPTURE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$") +_STATE_VALUES = frozenset(state.value for state in CaptureState) +_IMAGE_MEDIA_TYPES = frozenset({"image/png", "image/webp", "image/avif"}) +_OUTCOME_VALUES = frozenset({"updated", "not_found", "conflict", "retained"}) +_PortResult = TypeVar("_PortResult") + + +class ServiceUnavailable(RuntimeError): + """An injected port cannot safely complete this local request.""" + + +@dataclass(frozen=True, slots=True) +class WebConfig: + """Static local service settings supplied by runtime integration.""" + + server: ServerConfig + request_limits: RequestLimits = field(default_factory=RequestLimits) + max_capture_items: int = 64 + max_image_bytes: int = 64 * 1024 * 1024 + bootstrap_ttl_seconds: int = 600 + session_ttl_seconds: int = 900 + + def __post_init__(self) -> None: + for value, maximum in ( + (self.max_capture_items, 256), + (self.max_image_bytes, 64 * 1024 * 1024), + (self.bootstrap_ttl_seconds, 3600), + (self.session_ttl_seconds, 3600), + ): + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= maximum: + raise ValueError("web service configuration is invalid") + + @property + def authority(self) -> LocalAuthority: + return LocalAuthority(self.server.host, self.server.port) + + +@dataclass(frozen=True, slots=True) +class CaptureSessionStatus: + armed: bool + queue_depth: int + queue_capacity: int + + +@dataclass(frozen=True, slots=True) +class CaptureRecord: + """A safe browser projection; ports must never place paths or OCR text here.""" + + capture_id: str + state: str + received_at: str + classification: Mapping[str, object] | None + + +@dataclass(frozen=True, slots=True) +class MediaDescriptor: + """An opaque capture-media stream supplied by a corpus port, never a path.""" + + media_type: str + byte_size: int + chunks: Iterable[bytes] + + +MutationOutcome = Literal["updated", "not_found", "conflict", "retained"] +ControlOutcome = Literal["started", "already_armed", "stopped", "already_stopped", "conflict"] + + +class CaptureControlPort(Protocol): + """Narrow control boundary implemented by later runtime integration.""" + + def status(self) -> CaptureSessionStatus: ... + + def start_session(self) -> ControlOutcome: ... + + def stop_session(self) -> ControlOutcome: ... + + +class CaptureCorpusPort(Protocol): + """Narrow durable corpus boundary; routes never access SQLite or paths.""" + + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: ... + + def get_capture(self, capture_id: str) -> CaptureRecord | None: ... + + def open_media(self, capture_id: str) -> MediaDescriptor | None: ... + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: ... + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: ... + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: ... + + +def _response(payload: Mapping[str, object], *, status_code: int = 200) -> JSONResponse: + return JSONResponse( + payload, + status_code=status_code, + headers={"Cache-Control": "no-store", **response_security_headers()}, + ) + + +def _error(status_code: int, code: str) -> JSONResponse: + return _response({"error": code}, status_code=status_code) + + +def _call_port(operation: Callable[[], _PortResult]) -> _PortResult: + """Translate every injected-port failure without retaining its raw message.""" + + try: + return operation() + except Exception as error: + raise ServiceUnavailable("local service operation failed") from error + + +def _valid_capture_id(capture_id: str) -> bool: + return bool(_CAPTURE_ID.fullmatch(capture_id)) + + +def _status_projection(status: CaptureSessionStatus) -> dict[str, object]: + if ( + not isinstance(status.armed, bool) + or isinstance(status.queue_depth, bool) + or isinstance(status.queue_capacity, bool) + or not isinstance(status.queue_depth, int) + or not isinstance(status.queue_capacity, int) + or not 0 <= status.queue_depth <= status.queue_capacity <= 256 + ): + raise ServiceUnavailable("capture control returned an invalid status") + return { + "armed": status.armed, + "queue_depth": status.queue_depth, + "queue_capacity": status.queue_capacity, + } + + +def _classification_projection(classification: Mapping[str, object]) -> dict[str, object]: + # Reconstruct only the frozen contract keys before returning port data to a browser. + if not isinstance(classification, Mapping) or set(classification) != { + "family", + "view", + "gear_slot", + "record_types", + "confidence", + "abstention_reason", + }: + raise ServiceUnavailable("corpus returned an invalid classification") + payload = dict(classification) + try: + validate_classification(payload) + except ContractValidationError as error: + raise ServiceUnavailable("corpus returned an invalid classification") from error + family = payload["family"] + view = payload["view"] + gear_slot = payload["gear_slot"] + record_types = payload["record_types"] + confidence = payload["confidence"] + reason = payload["abstention_reason"] + if ( + not isinstance(family, str) + or not isinstance(view, str) + or (gear_slot is not None and not isinstance(gear_slot, str)) + or not isinstance(record_types, list) + or any(not isinstance(record_type, str) for record_type in record_types) + or isinstance(confidence, bool) + or not isinstance(confidence, int | float) + or (reason is not None and not isinstance(reason, str)) + ): + raise ServiceUnavailable("corpus returned an invalid classification") + return { + "family": family, + "view": view, + "gear_slot": gear_slot, + "record_types": record_types, + "confidence": confidence, + "abstention_reason": reason, + } + + +def _record_projection(record: CaptureRecord) -> dict[str, object]: + if ( + not isinstance(record.capture_id, str) + or not _valid_capture_id(record.capture_id) + or not isinstance(record.state, str) + or record.state not in _STATE_VALUES + or not isinstance(record.received_at, str) + or not 1 <= len(record.received_at) <= 40 + or any(character in record.received_at for character in "\r\n") + ): + raise ServiceUnavailable("corpus returned an invalid capture") + payload: dict[str, object] = { + "capture_id": record.capture_id, + "state": record.state, + "received_at": record.received_at, + "classification": None, + } + if record.classification is not None: + payload["classification"] = _classification_projection(record.classification) + return payload + + +def _json_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> object: + raise ValueError(f"unsupported JSON constant {value}") + + +async def _read_json(request: Request, *, maximum: int) -> dict[str, object]: + content_type = single_header(request, b"content-type", required=True) + if content_type is None: + raise SecurityError("unsupported request content type") + try: + media_type = content_type.decode("ascii") + except UnicodeDecodeError as error: + raise SecurityError("unsupported request content type") from error + if media_type.split(";", 1)[0].strip().lower() != "application/json": + raise SecurityError("unsupported request content type") + body = await request.body() + if not 2 <= len(body) <= maximum: + raise SecurityError("request JSON size is invalid") + try: + value = json.loads( + body.decode("utf-8"), + object_pairs_hook=_json_pairs, + parse_constant=_reject_json_constant, + ) + except ValueError as error: + raise SecurityError("request JSON is invalid") from error + if not isinstance(value, dict): + raise SecurityError("request JSON must be an object") + return value + + +def _bootstrap_payload(value: Mapping[str, object]) -> str: + if set(value) != {"token"}: + raise SecurityError("bootstrap payload is invalid") + token = value.get("token") + if not isinstance(token, str) or not 32 <= len(token) <= 128: + raise SecurityError("bootstrap payload is invalid") + return token + + +def _confirmation_payload(value: Mapping[str, object], confirmation: str) -> None: + if set(value) != {"confirmation"} or value.get("confirmation") != confirmation: + raise SecurityError("confirmation payload is invalid") + + +def _label_payload(value: Mapping[str, object]) -> dict[str, object]: + expected = { + "family", + "view", + "gear_slot", + "record_types", + "confidence", + "abstention_reason", + } + if set(value) != expected: + raise SecurityError("label payload is invalid") + family = value.get("family") + view = value.get("view") + record_types = value.get("record_types") + reason = value.get("abstention_reason") + if ( + not isinstance(family, str) + or not 1 <= len(family) <= 32 + or not isinstance(view, str) + or not 1 <= len(view) <= 32 + or not isinstance(record_types, list) + or not 0 <= len(record_types) <= 8 + or any( + not isinstance(record_type, str) or not 1 <= len(record_type) <= 64 + for record_type in record_types + ) + or (reason is not None and (not isinstance(reason, str) or not 1 <= len(reason) <= 64)) + ): + raise SecurityError("label payload is invalid") + payload = dict(value) + try: + validate_classification(payload) + except ContractValidationError as error: + raise SecurityError("label payload is invalid") from error + return payload + + +def _outcome_response(outcome: MutationOutcome) -> tuple[int, str]: + if outcome not in _OUTCOME_VALUES: + raise ServiceUnavailable("corpus returned an invalid mutation outcome") + if outcome == "updated": + return 200, "updated" + if outcome == "not_found": + return 404, "not_found" + if outcome == "conflict": + return 409, "conflict" + return 409, "retained" + + +def _media_bytes(media: MediaDescriptor, maximum: int) -> tuple[str, bytes]: + """Materialize and validate bounded port media before response headers exist.""" + + try: + if ( + media.media_type not in _IMAGE_MEDIA_TYPES + or isinstance(media.byte_size, bool) + or not isinstance(media.byte_size, int) + or not 1 <= media.byte_size <= maximum + ): + raise ServiceUnavailable("corpus returned unsafe media") + materialized = bytearray() + for chunk in media.chunks: + if not isinstance(chunk, bytes) or not chunk: + raise ServiceUnavailable("corpus returned unsafe media") + if ( + len(materialized) + len(chunk) > media.byte_size + or len(materialized) + len(chunk) > maximum + ): + raise ServiceUnavailable("corpus returned unsafe media") + materialized.extend(chunk) + if len(materialized) != media.byte_size: + raise ServiceUnavailable("corpus returned unsafe media") + return media.media_type, bytes(materialized) + except ServiceUnavailable: + raise + except Exception as error: + raise ServiceUnavailable("corpus media stream failed") from error + + +def _session_response( + payload: dict[str, object], + session_store: SessionStore, + session: AuthenticatedSession, + config: WebConfig, +) -> JSONResponse: + rotated = session_store.rotate(session) + payload["csrf_token"] = rotated.csrf_token + response = _response(payload) + set_session_cookie(response, rotated, max_age=config.session_ttl_seconds) + return response + + +def create_app( + *, + config: WebConfig, + capture_control: CaptureControlPort, + corpus: CaptureCorpusPort, + bootstrap_token: str, + now: Callable[[], float] = time.monotonic, +) -> FastAPI: + """Create an unbound app using only narrow, caller-injected local ports.""" + + authority = config.authority + sessions = SessionStore( + bootstrap_token, + now=now, + bootstrap_ttl_seconds=config.bootstrap_ttl_seconds, + session_ttl_seconds=config.session_ttl_seconds, + ) + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + app.add_middleware( + LocalRequestLimitsMiddleware, authority=authority, limits=config.request_limits + ) + app.mount( + "/static", + StaticFiles( + directory=Path(__file__).with_name("static"), + check_dir=False, + follow_symlink=False, + ), + name="static", + ) + templates = Jinja2Templates(directory=str(Path(__file__).with_name("templates"))) + + @app.exception_handler(AuthenticationError) + async def authentication_error(_: Request, __: AuthenticationError) -> JSONResponse: + return _error(401, "authentication_required") + + @app.exception_handler(BodyLimitExceeded) + async def body_limit_error(_: Request, __: BodyLimitExceeded) -> JSONResponse: + return _error(413, "request_too_large") + + @app.exception_handler(ClientDisconnected) + async def disconnect_error(_: Request, __: ClientDisconnected) -> JSONResponse: + return _error(400, "request_rejected") + + @app.exception_handler(SecurityError) + async def security_error(_: Request, __: SecurityError) -> JSONResponse: + return _error(403, "request_forbidden") + + @app.exception_handler(ServiceUnavailable) + async def service_error(_: Request, __: ServiceUnavailable) -> JSONResponse: + return _error(503, "service_unavailable") + + @app.exception_handler(Exception) + async def unexpected_error(_: Request, __: Exception) -> JSONResponse: + return _error(503, "service_unavailable") + + @app.get("/health") + async def health() -> JSONResponse: + return _response({"status": "ok"}) + + @app.get("/", response_class=Response) + async def review_page(request: Request) -> Response: + return templates.TemplateResponse(request=request, name="index.html", context={}) + + @app.post("/api/bootstrap") + async def bootstrap(request: Request) -> JSONResponse: + validate_origin(request, authority) + session = sessions.bootstrap(_bootstrap_payload(await _read_json(request, maximum=256))) + response = _response({"csrf_token": session.csrf_token}) + set_session_cookie(response, session, max_age=config.session_ttl_seconds) + return response + + @app.get("/api/session") + async def session_status(request: Request) -> JSONResponse: + session = sessions.authenticate(request) + payload = _status_projection(_call_port(capture_control.status)) + payload["csrf_token"] = session.csrf_token + return _response(payload) + + @app.post("/api/session/start") + async def start_session(request: Request) -> JSONResponse: + session = sessions.require_mutation(request, authority) + _confirmation_payload(await _read_json(request, maximum=64), "START") + outcome = _call_port(capture_control.start_session) + if outcome not in {"started", "already_armed", "conflict"}: + raise ServiceUnavailable("capture control returned an invalid start outcome") + if outcome == "conflict": + return _error(409, "conflict") + payload = _status_projection(_call_port(capture_control.status)) + payload["result"] = outcome + return _session_response(payload, sessions, session, config) + + @app.post("/api/session/stop") + async def stop_session(request: Request) -> JSONResponse: + session = sessions.require_mutation(request, authority) + _confirmation_payload(await _read_json(request, maximum=64), "STOP") + outcome = _call_port(capture_control.stop_session) + if outcome not in {"stopped", "already_stopped", "conflict"}: + raise ServiceUnavailable("capture control returned an invalid stop outcome") + if outcome == "conflict": + return _error(409, "conflict") + payload = _status_projection(_call_port(capture_control.status)) + payload["result"] = outcome + return _session_response(payload, sessions, session, config) + + @app.get("/api/captures") + async def list_captures(request: Request) -> JSONResponse: + sessions.authenticate(request) + + def list_projection() -> list[dict[str, object]]: + records = corpus.list_captures(limit=config.max_capture_items) + if len(records) > config.max_capture_items: + raise ValueError("corpus returned too many captures") + return [_record_projection(record) for record in records] + + return _response({"captures": _call_port(list_projection)}) + + @app.get("/api/captures/{capture_id}") + async def get_capture(capture_id: str, request: Request) -> JSONResponse: + sessions.authenticate(request) + if not _valid_capture_id(capture_id): + return _error(404, "not_found") + record = _call_port(lambda: corpus.get_capture(capture_id)) + if record is None: + return _error(404, "not_found") + return _response({"capture": _call_port(lambda: _record_projection(record))}) + + @app.get("/api/captures/{capture_id}/media") + async def capture_media(capture_id: str, request: Request) -> Response: + sessions.authenticate(request) + if not _valid_capture_id(capture_id): + return _error(404, "not_found") + media = _call_port(lambda: corpus.open_media(capture_id)) + if media is None: + return _error(404, "not_found") + media_type, content = _call_port(lambda: _media_bytes(media, config.max_image_bytes)) + return Response( + content=content, + media_type=media_type, + headers={ + "Cache-Control": "private, no-store", + }, + ) + + @app.post("/api/captures/{capture_id}/label") + async def label_capture(capture_id: str, request: Request) -> JSONResponse: + session = sessions.require_mutation(request, authority) + if not _valid_capture_id(capture_id): + return _error(404, "not_found") + classification = _label_payload(await _read_json(request, maximum=4096)) + outcome = _call_port(lambda: corpus.label(capture_id, classification)) + status, result = _outcome_response(outcome) + if status != 200: + return _error(status, result) + return _session_response({"result": result}, sessions, session, config) + + @app.post("/api/captures/{capture_id}/reject") + async def reject_capture(capture_id: str, request: Request) -> JSONResponse: + session = sessions.require_mutation(request, authority) + if not _valid_capture_id(capture_id): + return _error(404, "not_found") + _confirmation_payload(await _read_json(request, maximum=64), "REJECT") + outcome = _call_port(lambda: corpus.reject_preserving_evidence(capture_id)) + status, result = _outcome_response(outcome) + if status != 200: + return _error(status, result) + return _session_response({"result": result}, sessions, session, config) + + @app.post("/api/captures/{capture_id}/delete") + async def delete_capture(capture_id: str, request: Request) -> JSONResponse: + session = sessions.require_mutation(request, authority) + if not _valid_capture_id(capture_id): + return _error(404, "not_found") + _confirmation_payload(await _read_json(request, maximum=64), "DELETE") + outcome = _call_port(lambda: corpus.delete_with_retention(capture_id)) + status, result = _outcome_response(outcome) + if status != 200: + return _error(status, result) + return _session_response({"result": result}, sessions, session, config) + + return app diff --git a/tools/game-data-capture/src/game_data_capture/web/security.py b/tools/game-data-capture/src/game_data_capture/web/security.py new file mode 100644 index 0000000..26cffb6 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/web/security.py @@ -0,0 +1,352 @@ +"""Local-only request guards and process-local session state. + +This module intentionally has no runtime, storage, filesystem, or server imports. +The runtime integration layer owns listener construction and must disable Uvicorn +proxy headers so forwarded request headers never affect this boundary. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +import secrets +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from fastapi import Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +_SECRET = re.compile(r"^[A-Za-z0-9_-]{32,128}$") +_COOKIE_NAME = "capture_session" +_RESPONSE_SECURITY_HEADERS: tuple[tuple[bytes, bytes], ...] = ( + ( + b"content-security-policy", + b"default-src 'self'; base-uri 'none'; object-src 'none'; frame-src 'none'; " + b"frame-ancestors 'none'; " + b"script-src 'self'; style-src 'self'; font-src 'self'; img-src 'self'; " + b"media-src 'self'; connect-src 'self'; form-action 'self'", + ), + (b"referrer-policy", b"no-referrer"), + (b"x-content-type-options", b"nosniff"), + (b"x-frame-options", b"DENY"), +) + + +def response_security_headers() -> dict[str, str]: + """Return response headers for errors generated outside the ASGI wrapper.""" + + return { + name.decode("ascii"): value.decode("ascii") for name, value in _RESPONSE_SECURITY_HEADERS + } + + +class SecurityError(ValueError): + """A request failed a security boundary without exposing its input.""" + + +class AuthenticationError(SecurityError): + """The process-local authenticated session is absent or expired.""" + + +class BodyLimitExceeded(SecurityError): + """The declared or streamed request body exceeded the configured limit.""" + + +class ClientDisconnected(SecurityError): + """The client disconnected before a route obtained a complete request body.""" + + +@dataclass(frozen=True, slots=True) +class LocalAuthority: + """The single exact loopback authority accepted by the HTTP boundary.""" + + host: str + port: int + + def __post_init__(self) -> None: + try: + address = ipaddress.ip_address(self.host) + except ValueError as error: + raise ValueError("web host must be a loopback IP literal") from error + if not address.is_loopback or not 1024 <= self.port <= 65535: + raise ValueError("web authority is invalid") + + @property + def authority(self) -> str: + return f"[{self.host}]:{self.port}" if ":" in self.host else f"{self.host}:{self.port}" + + @property + def origin(self) -> str: + return f"http://{self.authority}" + + +@dataclass(frozen=True, slots=True) +class RequestLimits: + max_header_bytes: int = 16 * 1024 + max_headers: int = 64 + max_body_bytes: int = 16 * 1024 + + def __post_init__(self) -> None: + for value in (self.max_header_bytes, self.max_headers, self.max_body_bytes): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError("request limits must be positive integers") + + +def new_bootstrap_token() -> str: + """Create a fragment-safe, one-time token for the local bootstrap mutation.""" + + return secrets.token_urlsafe(32) + + +def _header_values(scope: Scope, name: bytes) -> list[bytes]: + return [value for key, value in scope.get("headers", []) if key.lower() == name] + + +def _single_header(scope: Scope, name: bytes, *, required: bool) -> bytes | None: + values = _header_values(scope, name) + if len(values) != 1: + if not values and not required: + return None + raise SecurityError("ambiguous request header") + return values[0] + + +def single_header(request: Request, name: bytes, *, required: bool) -> bytes | None: + """Return one unambiguous request header without trusting framework merging.""" + + return _single_header(request.scope, name, required=required) + + +def _ascii(value: bytes) -> str: + try: + return value.decode("ascii") + except UnicodeDecodeError as error: + raise SecurityError("invalid request header") from error + + +def validate_host(scope: Scope, authority: LocalAuthority) -> None: + value = _single_header(scope, b"host", required=True) + assert value is not None + if not secrets.compare_digest(_ascii(value), authority.authority): + raise SecurityError("unexpected request authority") + + +def validate_origin(request: Request, authority: LocalAuthority) -> None: + value = _single_header(request.scope, b"origin", required=True) + assert value is not None + if not secrets.compare_digest(_ascii(value), authority.origin): + raise SecurityError("unexpected request origin") + + +def _validate_content_length(scope: Scope, maximum: int) -> None: + value = _single_header(scope, b"content-length", required=False) + if value is None: + return + raw = _ascii(value) + if not raw.isascii() or not raw.isdecimal(): + raise SecurityError("invalid content length") + if int(raw) > maximum: + raise BodyLimitExceeded("request body is too large") + + +async def _send_rejection(send: Send, status: int, code: str) -> None: + body = json.dumps({"error": code}, separators=(",", ":")).encode("ascii") + await send( + { + "type": "http.response.start", + "status": status, + "headers": [ + (b"content-type", b"application/json"), + (b"cache-control", b"no-store"), + (b"content-length", str(len(body)).encode("ascii")), + ], + } + ) + await send({"type": "http.response.body", "body": body}) + + +class LocalRequestLimitsMiddleware: + """Validate headers and cap all ASGI body chunks before application parsing.""" + + def __init__(self, app: ASGIApp, *, authority: LocalAuthority, limits: RequestLimits) -> None: + self.app = app + self.authority = authority + self.limits = limits + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def secured_send(message: Message) -> None: + if message["type"] != "http.response.start": + await send(message) + return + headers = list(message.get("headers", [])) + present = {name.lower() for name, _ in headers} + headers.extend( + (name, value) for name, value in _RESPONSE_SECURITY_HEADERS if name not in present + ) + protected = dict(message) + protected["headers"] = headers + await send(protected) + + headers = scope.get("headers", []) + if len(headers) > self.limits.max_headers or any( + len(name) + len(value) > self.limits.max_header_bytes for name, value in headers + ): + await _send_rejection(secured_send, 431, "request_rejected") + return + if sum(len(name) + len(value) for name, value in headers) > self.limits.max_header_bytes: + await _send_rejection(secured_send, 431, "request_rejected") + return + try: + validate_host(scope, self.authority) + _validate_content_length(scope, self.limits.max_body_bytes) + except BodyLimitExceeded: + await _send_rejection(secured_send, 413, "request_too_large") + return + except SecurityError: + await _send_rejection(secured_send, 400, "request_rejected") + return + + received = 0 + request_complete = False + + async def limited_receive() -> Message: + nonlocal received, request_complete + message = await receive() + if message["type"] == "http.disconnect": + if not request_complete: + raise ClientDisconnected("client disconnected") + return message + if message["type"] != "http.request": + return message + body = message.get("body", b"") + if not isinstance(body, bytes): + raise SecurityError("invalid request body") + received += len(body) + if received > self.limits.max_body_bytes: + raise BodyLimitExceeded("request body is too large") + if not message.get("more_body", False): + request_complete = True + return message + + await self.app(scope, limited_receive, secured_send) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedSession: + token: str + csrf_token: str + expires_at: float + + +class SessionStore: + """An intentionally non-durable session store invalidated on process restart.""" + + def __init__( + self, + bootstrap_token: str, + *, + now: Callable[[], float] = time.monotonic, + bootstrap_ttl_seconds: int = 600, + session_ttl_seconds: int = 900, + ) -> None: + if not isinstance(bootstrap_token, str) or not _SECRET.fullmatch(bootstrap_token): + raise ValueError("bootstrap token is invalid") + if ( + isinstance(bootstrap_ttl_seconds, bool) + or not isinstance(bootstrap_ttl_seconds, int) + or isinstance(session_ttl_seconds, bool) + or not isinstance(session_ttl_seconds, int) + or not 1 <= bootstrap_ttl_seconds <= 3600 + or not 1 <= session_ttl_seconds <= 3600 + ): + raise ValueError("session lifetime is invalid") + self._bootstrap_token = bootstrap_token.encode("ascii") + self._bootstrap_expires_at = now() + bootstrap_ttl_seconds + self._bootstrap_used = False + self._now = now + self._session_ttl_seconds = session_ttl_seconds + self._sessions: dict[str, AuthenticatedSession] = {} + + def bootstrap(self, submitted: str) -> AuthenticatedSession: + if ( + self._bootstrap_used + or self._now() > self._bootstrap_expires_at + or not _SECRET.fullmatch(submitted) + or not secrets.compare_digest(submitted.encode("ascii"), self._bootstrap_token) + ): + raise AuthenticationError("bootstrap authentication failed") + self._bootstrap_used = True + return self._new_session() + + def authenticate(self, request: Request) -> AuthenticatedSession: + token = _session_cookie(request) + session = self._sessions.get(token) + if session is None or self._now() > session.expires_at: + self._sessions.pop(token, None) + raise AuthenticationError("session authentication failed") + return session + + def require_mutation(self, request: Request, authority: LocalAuthority) -> AuthenticatedSession: + session = self.authenticate(request) + validate_origin(request, authority) + csrf = _single_header(request.scope, b"x-csrf-token", required=True) + assert csrf is not None + submitted = _ascii(csrf) + if not _SECRET.fullmatch(submitted) or not secrets.compare_digest( + submitted.encode("ascii"), session.csrf_token.encode("ascii") + ): + raise SecurityError("csrf validation failed") + return session + + def rotate(self, session: AuthenticatedSession) -> AuthenticatedSession: + current = self._sessions.get(session.token) + if current != session or self._now() > session.expires_at: + self._sessions.pop(session.token, None) + raise AuthenticationError("session authentication failed") + del self._sessions[session.token] + return self._new_session() + + def _new_session(self) -> AuthenticatedSession: + session = AuthenticatedSession( + token=secrets.token_urlsafe(32), + csrf_token=secrets.token_urlsafe(32), + expires_at=self._now() + self._session_ttl_seconds, + ) + self._sessions[session.token] = session + return session + + +def _session_cookie(request: Request) -> str: + raw = _single_header(request.scope, b"cookie", required=True) + assert raw is not None + pairs: dict[str, str] = {} + for segment in _ascii(raw).split(";"): + name, separator, value = segment.strip().partition("=") + if not separator or not name or name in pairs: + raise AuthenticationError("session authentication failed") + pairs[name] = value + token = pairs.get(_COOKIE_NAME) + if token is None or not _SECRET.fullmatch(token): + raise AuthenticationError("session authentication failed") + return token + + +def set_session_cookie(response: Any, session: AuthenticatedSession, *, max_age: int) -> None: + """Set a host-only HTTP-only local session cookie without a Domain attribute.""" + + response.set_cookie( + key=_COOKIE_NAME, + value=session.token, + max_age=max_age, + path="/", + secure=False, + httponly=True, + samesite="strict", + ) diff --git a/tools/game-data-capture/src/game_data_capture/web/static/review.css b/tools/game-data-capture/src/game_data_capture/web/static/review.css new file mode 100644 index 0000000..0bb082f --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/web/static/review.css @@ -0,0 +1,124 @@ +:root { + --canvas: #0a0c0d; + --surface: #111415; + --surface-raised: #141718; + --well: #0d0f10; + --line: #303536; + --line-strong: #50595b; + --text: #ecebe5; + --muted: #a8afb1; + --dim: #7e8789; + --orange: #f6a821; + --orange-hover: #ffc45d; + --success: #8bd2a0; + --danger: #f09a86; + --info: #8cc8df; + --focus: #ffc45d; + --space: 1rem; + font-family: Arial, Helvetica, sans-serif; + color: var(--text); + background: var(--canvas); +} + +* { box-sizing: border-box; } + +body { margin: 0; min-width: 20rem; background: var(--canvas); color: var(--text); } + +button, input { font: inherit; } + +button, input[type="radio"], input[type="checkbox"] { accent-color: var(--orange); } + +button { min-height: 44px; border: 1px solid var(--line-strong); border-radius: 0; background: var(--surface); color: var(--text); cursor: pointer; padding: .65rem .9rem; font-weight: 700; } +button:hover:not(:disabled) { background: #1b2021; border-color: var(--text); } +button:active:not(:disabled) { transform: translateY(1px); } +button:disabled { color: var(--dim); cursor: not-allowed; border-color: var(--line); background: #101213; } +.button-primary { background: var(--orange); color: #12100b; border-color: var(--orange); } +.button-primary:hover:not(:disabled) { background: var(--orange-hover); border-color: var(--orange-hover); } +.button-danger { color: var(--danger); } +.quiet-danger { border-color: #6f4941; } + +:focus-visible { outline: 3px solid var(--focus); outline-offset: 3px; } + +.skip-link { position: fixed; z-index: 2; top: 0; left: .5rem; transform: translateY(-120%); background: var(--orange); color: #12100b; padding: .65rem; font-weight: 700; } +.skip-link:focus { transform: translateY(0); } + +.shell { width: min(100% - 2rem, 100rem); margin: 0 auto; padding: 1.5rem 0 3rem; } +.command-header, .control-strip, .panel-heading { display: flex; gap: var(--space); justify-content: space-between; align-items: center; } +.command-header { border-bottom: 1px solid var(--line); padding: 0 0 1.25rem; } +.kicker { margin: 0 0 .35rem; color: var(--muted); font-size: .74rem; font-weight: 800; letter-spacing: .08em; } +h1, h2, p { margin-top: 0; } +h1 { margin-bottom: 0; font-size: clamp(1.65rem, 3vw, 2.35rem); letter-spacing: -.025em; } +h2 { margin-bottom: 0; font-size: 1.15rem; } +.header-state { text-align: right; } +.header-state strong, #capture-state, #review-state { display: block; font-size: .88rem; font-weight: 800; letter-spacing: .04em; } +.header-state span, #queue-state, #capture-count, .field-help, .action-help, figcaption { color: var(--muted); font-size: .88rem; } + +.notice { display: flex; gap: .75rem; border: 1px solid var(--line); border-left: 2px solid var(--info); margin: 1rem 0; padding: .8rem 1rem; background: var(--well); } +.notice strong { color: var(--info); font-size: .78rem; letter-spacing: .06em; } +.notice[data-kind="success"] { border-left-color: var(--success); } +.notice[data-kind="success"] strong { color: var(--success); } +.notice[data-kind="error"], .notice[data-kind="conflict"] { border-left-color: var(--danger); } +.notice[data-kind="error"] strong, .notice[data-kind="conflict"] strong { color: var(--danger); } +.notice[data-kind="retained"] { border-left-color: var(--info); } +.notice[data-kind="retained"] strong { color: var(--info); } + +.control-strip { border: 1px solid var(--line); background: var(--surface-raised); padding: 1rem; margin-bottom: 1rem; } +.control-strip p:last-child { margin: 0; } +.control-actions, .form-actions, .dialog-actions { display: flex; gap: .5rem; flex-wrap: wrap; } + +.workspace-grid { display: grid; grid-template-columns: minmax(17rem, 24rem) minmax(0, 1fr); border: 1px solid var(--line); background: var(--surface); } +.capture-panel { border-right: 1px solid var(--line); min-height: 42rem; } +.capture-panel, .review-panel { padding: 1rem; } +.panel-heading { border-bottom: 1px solid var(--line); padding-bottom: .85rem; } +.empty-copy, .review-empty { color: var(--muted); padding: 1rem 0; line-height: 1.5; } +.capture-list { margin: 0; padding: 0; list-style: none; } +.capture-list button { width: 100%; display: grid; grid-template-columns: 1fr auto; gap: .25rem .6rem; margin-top: .55rem; text-align: left; background: var(--well); } +.capture-list button[aria-current="true"] { border-color: var(--orange); color: var(--text); } +.capture-list .capture-id { overflow-wrap: anywhere; } +.capture-list .capture-meta { color: var(--muted); font-size: .8rem; } + +.review-content { padding-top: 1rem; } +.evidence-frame { margin: 0 0 1rem; border: 1px solid var(--line); background: var(--well); padding: .65rem; } +.evidence-frame img { display: block; width: 100%; max-height: 31rem; object-fit: contain; background: #000; } +.evidence-frame figcaption { padding-top: .65rem; overflow-wrap: anywhere; } +fieldset { margin: 0 0 1rem; padding: .75rem; border: 1px solid var(--line); min-width: 0; } +fieldset:disabled { opacity: .65; } +legend { padding: 0 .35rem; font-weight: 700; } +.choice-grid { display: flex; flex-wrap: wrap; gap: .45rem; } +.choice { position: relative; display: inline-flex; min-height: 44px; align-items: center; border: 1px solid var(--line-strong); background: var(--well); cursor: pointer; } +.choice:hover { border-color: var(--text); } +.choice input { position: absolute; opacity: 0; inset: 0; cursor: pointer; } +.choice span { padding: .65rem .75rem; } +.choice input:checked + span { background: #3c2a0c; color: var(--text); box-shadow: inset 0 -2px var(--orange); } +.choice input:focus-visible + span { outline: 3px solid var(--focus); outline-offset: 2px; } +.validation-message { min-height: 1.5rem; margin: -.35rem 0 .75rem; color: var(--danger); } +.action-help { max-width: 70ch; margin: .8rem 0 0; line-height: 1.45; } + +dialog { width: min(calc(100% - 2rem), 34rem); border: 1px solid var(--orange); border-radius: 0; padding: 1.3rem; color: var(--text); background: var(--surface-raised); box-shadow: 0 18px 60px #000; } +dialog::backdrop { background: rgb(0 0 0 / 72%); } +dialog p { line-height: 1.5; } + +@media (max-width: 52rem) { + .shell { width: min(100% - 1rem, 100rem); padding-top: 1rem; } + .command-header, .control-strip { align-items: flex-start; flex-direction: column; } + .header-state { text-align: left; } + .workspace-grid { grid-template-columns: 1fr; } + .capture-panel { min-height: auto; border-right: 0; border-bottom: 1px solid var(--line); } + .capture-list { display: grid; grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr)); gap: .55rem; } + .capture-list button { margin-top: 0; } +} + +@media (max-width: 33rem) { + .control-actions, .form-actions, .dialog-actions { display: grid; grid-template-columns: 1fr; width: 100%; } + .control-actions button, .form-actions button, .dialog-actions button { width: 100%; } + .notice { align-items: flex-start; flex-direction: column; gap: .25rem; } +} + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { scroll-behavior: auto !important; transition-duration: .01ms !important; animation-duration: .01ms !important; animation-iteration-count: 1 !important; } +} + +@media (forced-colors: active) { + button, fieldset, .choice, .notice, .workspace-grid, .evidence-frame { border: 1px solid CanvasText; } + .button-primary, .choice input:checked + span { forced-color-adjust: none; background: Highlight; color: HighlightText; } +} diff --git a/tools/game-data-capture/src/game_data_capture/web/static/review.js b/tools/game-data-capture/src/game_data_capture/web/static/review.js new file mode 100644 index 0000000..87e099a --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/web/static/review.js @@ -0,0 +1,286 @@ +const taxonomy = Object.freeze({ + weapon: { views: ["detail", "blueprint", "crafting", "stat_range", "talent_grid", "augmentation"], records: { detail: ["weapon", "weapon_talent"], blueprint: ["weapon", "weapon_talent"], crafting: ["weapon"], stat_range: ["weapon"], talent_grid: ["weapon_talent"], augmentation: ["weapon_augmentation_profile"] } }, + gear: { views: ["detail", "blueprint", "talent", "augmentation"], records: { detail: ["gear", "brand_set"], blueprint: ["gear", "brand_set"], talent: ["gear_talent"], augmentation: ["gear_augmentation_profile"] } }, + skill_chip: { views: ["detail", "set_bonus", "talent"], records: { detail: ["skill_chip_set", "skill_chip_roll_pool"], set_bonus: ["skill_chip_set_bonus"], talent: ["skill_chip_talent"] } }, + os_protocol: { views: ["detail", "secondary_profile", "talent"], records: { detail: ["os_protocol"], secondary_profile: ["os_protocol_secondary_profile"], talent: ["os_protocol_talent"] } }, + specialization: { views: ["overview", "details", "ability", "focus_details", "focus_tree", "rank_details"], records: { overview: ["specialization"], details: ["specialization"], ability: ["specialization_ability"], focus_details: ["specialization_focus"], focus_tree: ["specialization_focus", "specialization_tree_node"], rank_details: ["specialization_tree_rank"] } }, + unknown: { views: ["unknown"], records: { unknown: [] } } +}); +const families = Object.keys(taxonomy); +const gearSlots = ["Mask", "Gloves", "Backpack", "Body Armor", "Holster", "Knee Pads"]; +const abstentionReasons = ["low_confidence", "ambiguous_family", "ambiguous_view", "unsupported_family", "unsupported_view", "insufficient_visual_evidence", "occluded", "corrupt_image", "resource_limit", "cancelled"]; +const csrfPattern = /^[A-Za-z0-9_-]{32,128}$/; + +const $ = (id) => document.getElementById(id); +const els = Object.freeze({ + sessionState: $("session-state"), queueState: $("queue-state"), captureState: $("capture-state"), notice: $("notice"), noticeTitle: $("notice-title"), noticeCopy: $("notice-copy"), start: $("start-button"), stop: $("stop-button"), refresh: $("refresh-button"), list: $("capture-list"), listEmpty: $("capture-empty"), count: $("capture-count"), reviewTitle: $("review-title"), reviewState: $("review-state"), reviewEmpty: $("review-empty"), reviewContent: $("review-content"), media: $("capture-media"), meta: $("capture-meta"), labelForm: $("label-form"), validation: $("label-validation"), save: $("save-label-button"), reject: $("reject-button"), remove: $("delete-button"), family: $("family-options"), view: $("view-options"), gear: $("gear-slot-options"), records: $("record-type-options"), abstention: $("abstention-options"), viewFieldset: $("view-fieldset"), gearFieldset: $("gear-slot-fieldset"), recordFieldset: $("record-types-fieldset"), abstentionFieldset: $("abstention-fieldset"), dialog: $("confirmation-dialog"), dialogKicker: $("confirmation-kicker"), dialogTitle: $("confirmation-title"), dialogCopy: $("confirmation-copy"), dialogSubmit: $("confirmation-submit") +}); + +let csrfToken = null; +let activeCapture = null; +let captures = []; +let armed = null; +let pending = false; +let returnFocus = null; + +function setText(element, value) { element.textContent = value; } +function clearNode(node) { while (node.firstChild) node.removeChild(node.firstChild); } +function label(value) { return value.replaceAll("_", " "); } +function notice(kind, title, copy) { els.notice.dataset.kind = kind; setText(els.noticeTitle, title); setText(els.noticeCopy, copy); } +function hasValidCsrf(value) { return typeof value === "string" && csrfPattern.test(value); } +function selected(name) { return document.querySelector(`input[name="${name}"]:checked`)?.value || null; } +function selectedAll(name) { return [...document.querySelectorAll(`input[name="${name}"]:checked`)].map((input) => input.value); } +function currentFamily() { return selected("family"); } +function currentView() { return selected("view"); } + +function syncControls() { + const locked = !csrfToken; + els.start.disabled = pending || locked || armed === true; + els.stop.disabled = pending || locked || armed !== true; + els.refresh.disabled = pending || locked; + els.save.disabled = pending || locked || !activeCapture || !isValidLabel(); + els.reject.disabled = pending || locked || !activeCapture; + els.remove.disabled = pending || locked || !activeCapture; + document.querySelectorAll("#capture-list button").forEach((button) => { + button.disabled = pending || locked; + }); + document.querySelectorAll("#label-form input").forEach((input) => { + const fieldset = input.closest("fieldset"); + input.disabled = pending || locked || !activeCapture || Boolean(fieldset?.disabled || fieldset?.hidden); + }); + els.labelForm.setAttribute("aria-busy", String(pending)); +} +function setBusy(value) { pending = value; syncControls(); } +function clearActiveDetail() { + activeCapture = null; + els.media.removeAttribute("src"); + els.reviewContent.hidden = true; + els.reviewEmpty.hidden = false; + els.reviewTitle.textContent = "No capture selected"; + els.reviewState.textContent = csrfToken ? "READY" : "LOCKED"; + renderLabelForm(null); +} +function lock(message) { + csrfToken = null; + armed = null; + captures = []; + clearActiveDetail(); + els.sessionState.textContent = "LOCKED"; + els.captureState.textContent = "DISARMED"; + els.queueState.textContent = "Queue unavailable"; + els.listEmpty.hidden = false; + setText(els.listEmpty, message); + clearNode(els.list); + els.count.textContent = "0 items"; + setBusy(false); +} +function validSession(payload) { + return Boolean(payload) && hasValidCsrf(payload.csrf_token) && typeof payload.armed === "boolean" && Number.isInteger(payload.queue_depth) && Number.isInteger(payload.queue_capacity) && payload.queue_depth >= 0 && payload.queue_depth <= payload.queue_capacity && payload.queue_capacity <= 256; +} +function updateSession(payload) { + armed = payload.armed; + els.sessionState.textContent = "UNLOCKED"; + els.captureState.textContent = armed ? "ARMED" : "DISARMED"; + els.queueState.textContent = `Queue ${payload.queue_depth} / ${payload.queue_capacity}`; + if (!activeCapture) els.reviewState.textContent = "READY"; + syncControls(); +} +function safeJson(response) { return response.json().catch(() => null); } +async function request(path, options = {}, mutation = false) { + const headers = new Headers(options.headers || {}); + if (mutation) { headers.set("Content-Type", "application/json"); headers.set("X-CSRF-Token", csrfToken || ""); } + let response; + try { response = await fetch(path, { ...options, headers, credentials: "same-origin" }); } catch { notice("error", "NETWORK ERROR", "The local service did not respond. Your current review remains unchanged."); return null; } + if (response.status === 401 || response.status === 403) { lock("Local session expired or restarted. Open the provided local address again."); notice("error", "SESSION LOCKED", "The local session is no longer available. No action was retried."); return null; } + const payload = await safeJson(response); + if (payload === null) { notice("error", "RESPONSE ERROR", "The local service returned an unusable response. Your current review remains unchanged."); return null; } + if (!response.ok) { + const retained = response.status === 409 && payload.error === "retained"; + notice(retained ? "retained" : response.status === 409 ? "conflict" : "error", retained ? "RETAINED" : response.status === 409 ? "CONFLICT" : "ACTION FAILED", retained ? "Retention policy kept this evidence. No deletion occurred." : "The local service could not complete that action."); + return null; + } + return payload; +} +function acceptMutationCsrf(payload) { + if (hasValidCsrf(payload?.csrf_token)) { csrfToken = payload.csrf_token; return true; } + lock("The local service returned an invalid session update. Open the provided local address again."); + notice("error", "SESSION LOCKED", "The confirmed action was not retried."); + return false; +} +function addChoice(container, type, name, value, checked, onChange) { + const wrapper = document.createElement("label"); wrapper.className = "choice"; + const input = document.createElement("input"); input.type = type; input.name = name; input.value = value; input.checked = checked; input.addEventListener("change", onChange); + const text = document.createElement("span"); text.textContent = label(value); + wrapper.append(input, text); container.append(wrapper); +} +function setChecked(name, value) { + document.querySelectorAll(`input[name="${name}"]`).forEach((input) => { input.checked = input.value === value; }); +} +function setCheckedMembers(name, values) { + const selectedValues = new Set(values); + document.querySelectorAll(`input[name="${name}"]`).forEach((input) => { + input.checked = selectedValues.has(input.value); + }); +} +function renderFamilies() { + clearNode(els.family); + families.forEach((family) => addChoice(els.family, "radio", "family", family, false, () => { renderViews(); renderDependentFields(); })); +} +function renderViews() { + const family = currentFamily(); + clearNode(els.view); + els.viewFieldset.disabled = !family; + if (!family) return; + taxonomy[family].views.forEach((view) => addChoice(els.view, "radio", "view", view, false, renderDependentFields)); +} +function renderDependentFields() { + const family = currentFamily(); const view = currentView(); const unknown = family === "unknown"; const isGear = family === "gear"; + els.gearFieldset.hidden = !isGear; els.gearFieldset.disabled = !isGear; + els.abstentionFieldset.hidden = !unknown; els.abstentionFieldset.disabled = !unknown; + els.recordFieldset.disabled = !family || !view || unknown; + clearNode(els.gear); clearNode(els.records); clearNode(els.abstention); + if (isGear) gearSlots.forEach((slot) => addChoice(els.gear, "radio", "gear_slot", slot, false, validateLabel)); + if (unknown) abstentionReasons.forEach((reason) => addChoice(els.abstention, "radio", "abstention_reason", reason, false, validateLabel)); + if (!unknown && family && view) (taxonomy[family].records[view] || []).forEach((record) => addChoice(els.records, "checkbox", "record_type", record, false, validateLabel)); + validateLabel(); +} +function labelPayload() { + const family = currentFamily(); const view = currentView(); const unknown = family === "unknown"; + return { family, view, gear_slot: family === "gear" ? selected("gear_slot") : null, record_types: unknown ? [] : selectedAll("record_type"), confidence: 1.0, abstention_reason: unknown ? selected("abstention_reason") : null }; +} +function isValidLabel() { + const payload = labelPayload(); + if (!payload.family || !payload.view) return false; + if (payload.family === "unknown") return payload.view === "unknown" && !payload.gear_slot && payload.record_types.length === 0 && Boolean(payload.abstention_reason); + return payload.record_types.length > 0 && (payload.family !== "gear" || Boolean(payload.gear_slot)); +} +function validateLabel() { + const payload = labelPayload(); let message = ""; + if (!payload.family) message = "Select the evidence family."; + else if (!payload.view) message = "Select the screen view."; + else if (payload.family === "unknown" && !payload.abstention_reason) message = "Select an explicit abstention reason for an unknown screen."; + else if (payload.family === "gear" && !payload.gear_slot) message = "Select the Gear slot shown by this evidence."; + else if (payload.family !== "unknown" && payload.record_types.length === 0) message = "Select at least one compatible canonical record type."; + setText(els.validation, message); + if (!pending) syncControls(); + return !message; +} +function renderLabelForm(classification) { + renderFamilies(); + if (!classification) { renderViews(); renderDependentFields(); return; } + setChecked("family", classification.family); + renderViews(); + setChecked("view", classification.view); + renderDependentFields(); + if (classification.gear_slot) setChecked("gear_slot", classification.gear_slot); + setCheckedMembers("record_type", classification.record_types || []); + if (classification.abstention_reason) setChecked("abstention_reason", classification.abstention_reason); + validateLabel(); +} +function renderCaptures() { + clearNode(els.list); els.count.textContent = `${captures.length} ${captures.length === 1 ? "item" : "items"}`; els.listEmpty.hidden = captures.length > 0; + if (!captures.length) { setText(els.listEmpty, "EMPTY QUEUE: capture evidence will appear after a manual refresh."); return; } + captures.forEach((capture) => { + const item = document.createElement("li"); const button = document.createElement("button"); button.type = "button"; button.setAttribute("aria-current", String(activeCapture?.capture_id === capture.capture_id)); + const id = document.createElement("span"); id.className = "capture-id"; id.textContent = capture.capture_id; + const meta = document.createElement("span"); meta.className = "capture-meta"; meta.textContent = `${capture.state} · ${capture.received_at}`; + button.append(id, meta); button.addEventListener("click", () => selectCapture(capture.capture_id)); item.append(button); els.list.append(item); + }); + syncControls(); +} +function focusIfAvailable(control) { if (control && !control.disabled) control.focus(); } +async function refreshCaptures({ announce = true, fallbackFocus = null } = {}) { + if (!csrfToken || pending) return false; + setBusy(true); if (announce) notice("info", "LOADING", "Refreshing the local capture queue."); + const payload = await request("/api/captures"); + if (!payload || !Array.isArray(payload.captures)) { setBusy(false); if (announce) focusIfAvailable(fallbackFocus); return false; } + captures = payload.captures; + if (activeCapture && !captures.some((capture) => capture.capture_id === activeCapture.capture_id)) clearActiveDetail(); + renderCaptures(); if (announce) notice("success", "QUEUE REFRESHED", "Local capture list updated."); setBusy(false); if (announce) focusIfAvailable(fallbackFocus); return true; +} +async function selectCapture(captureId, { announce = true } = {}) { + if (!csrfToken || pending) return; + setBusy(true); if (announce) notice("info", "LOADING", "Loading selected private evidence."); + const payload = await request(`/api/captures/${encodeURIComponent(captureId)}`); + if (!payload?.capture) { setBusy(false); return; } + activeCapture = payload.capture; els.reviewContent.hidden = false; els.reviewEmpty.hidden = true; els.reviewTitle.textContent = activeCapture.capture_id; els.reviewState.textContent = activeCapture.state; + els.meta.textContent = `State: ${activeCapture.state} · Received: ${activeCapture.received_at}`; + els.media.removeAttribute("src"); els.media.src = `/api/captures/${encodeURIComponent(activeCapture.capture_id)}/media`; + renderCaptures(); renderLabelForm(activeCapture.classification); if (announce) notice("success", "CAPTURE READY", "Review the evidence, then save a manual label or choose a separate evidence action."); setBusy(false); +} +function routeForAction(action) { + if (action === "start") return ["/api/session/start", "START"]; + if (action === "stop") return ["/api/session/stop", "STOP"]; + if ((action === "reject" || action === "delete") && activeCapture) return [`/api/captures/${encodeURIComponent(activeCapture.capture_id)}/${action}`, action === "reject" ? "REJECT" : "DELETE"]; + return null; +} +function showConfirmation(action, source) { + if ((action === "reject" || action === "delete") && !activeCapture) { notice("error", "ACTION UNAVAILABLE", "Select a capture before confirming this evidence action."); return; } + const messages = { start: ["START SESSION", "Start capture now? This is the only action that can arm capture.", "Start Session"], stop: ["STOP SESSION", "Stop capture now? New screenshots will not enter the queue after the session stops.", "Stop Session"], reject: ["REJECT EVIDENCE", "Reject this evidence? The evidence remains retained for review and will not be deleted.", "Reject evidence"], delete: ["DELETE CAPTURE", "Request deletion of this capture? Retention policy may refuse deletion. Rejection does not delete evidence.", "Delete capture"] }; + const message = messages[action]; if (!message) return; + returnFocus = source; els.dialog.returnValue = "cancel"; els.dialog.dataset.action = action; + els.dialogKicker.textContent = "EXPLICIT CONFIRMATION"; els.dialogTitle.textContent = message[0]; els.dialogCopy.textContent = message[1]; els.dialogSubmit.textContent = message[2]; els.dialogSubmit.className = action === "delete" || action === "reject" ? "button button-danger" : "button button-primary"; + els.dialog.showModal(); els.dialogSubmit.focus(); +} +async function confirmed(action, fallbackFocus) { + if (pending) return; + const route = routeForAction(action); + if (!route) { notice("error", "ACTION UNAVAILABLE", "Select a capture before confirming this evidence action."); fallbackFocus?.focus(); return; } + setBusy(true); notice("info", "PENDING", "The local service is processing your confirmed action."); + const [path, confirmation] = route; const payload = await request(path, { method: "POST", body: JSON.stringify({ confirmation }) }, true); + if (!payload || !acceptMutationCsrf(payload)) { setBusy(false); fallbackFocus?.focus(); return; } + if ((action === "start" || action === "stop") && !validSession(payload)) { lock("The local service returned an invalid session update. Open the provided local address again."); setBusy(false); return; } + if (action === "delete") clearActiveDetail(); + if (action === "start" || action === "stop") updateSession(payload); + const successFocus = action === "start" ? els.stop : action === "stop" ? els.start : action === "delete" ? els.refresh : fallbackFocus; + setBusy(false); if (!(await refreshCaptures({ announce: false }))) { focusIfAvailable(successFocus); return; } + notice("success", "SUCCESS", action === "reject" ? "Evidence rejected and retained." : action === "delete" ? "Deletion request completed." : `Capture session ${action === "start" ? "armed" : "disarmed"}.`); + focusIfAvailable(successFocus); +} +async function initialize() { + renderLabelForm(null); + let fragment = window.location.hash; let token = null; + if (fragment.length > 1 && fragment.length <= 256) { const params = new URLSearchParams(fragment.slice(1)); const candidate = params.get("bootstrap"); if (candidate && csrfPattern.test(candidate)) token = candidate; } + if (fragment) history.replaceState(null, "", `${window.location.pathname}${window.location.search}`); + fragment = ""; + if (token) { + const bootstrap = await request("/api/bootstrap", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ token }) }); token = null; + if (!hasValidCsrf(bootstrap?.csrf_token)) { lock("The local service could not establish a safe session. Open the provided local address again."); return; } + csrfToken = bootstrap.csrf_token; + } + const session = await request("/api/session"); + if (!validSession(session)) { lock("LOCKED: start the local service with its provided bootstrap address. This browser has no process-local session."); return; } + csrfToken = session.csrf_token; updateSession(session); if (!(await refreshCaptures({ announce: false }))) return; + const startupCopy = session.armed ? "Capture is ARMED. Stop Session requires explicit confirmation." : "Capture is DISARMED. Start Session requires explicit confirmation."; + notice("success", "LOCAL SESSION READY", startupCopy); +} + +els.start.addEventListener("click", () => showConfirmation("start", els.start)); +els.stop.addEventListener("click", () => showConfirmation("stop", els.stop)); +els.refresh.addEventListener("click", () => refreshCaptures({ fallbackFocus: els.refresh })); +els.labelForm.addEventListener("submit", async (event) => { + event.preventDefault(); + if (!activeCapture || !validateLabel() || pending) return; + const captureId = activeCapture.capture_id; + setBusy(true); notice("info", "PENDING", "Saving the manual reviewed label."); + const payload = await request(`/api/captures/${encodeURIComponent(captureId)}/label`, { method: "POST", body: JSON.stringify(labelPayload()) }, true); + if (!payload || !acceptMutationCsrf(payload)) { setBusy(false); return; } + setBusy(false); + if (!(await refreshCaptures({ announce: false }))) { focusIfAvailable(els.save); return; } + if (captures.some((capture) => capture.capture_id === captureId)) await selectCapture(captureId, { announce: false }); + notice("success", "LABEL SAVED", "Manual reviewed label saved with confidence 1.0."); + focusIfAvailable(els.save); +}); +els.reject.addEventListener("click", () => showConfirmation("reject", els.reject)); +els.remove.addEventListener("click", () => showConfirmation("delete", els.remove)); +els.dialog.addEventListener("close", () => { + const action = els.dialog.dataset.action; const shouldConfirm = els.dialog.returnValue === "confirm"; + els.dialog.dataset.action = ""; + const focusTarget = returnFocus; + if (focusTarget) focusTarget.focus(); + returnFocus = null; + if (shouldConfirm && action) confirmed(action, focusTarget); +}); +els.dialog.addEventListener("cancel", () => { els.dialog.returnValue = "cancel"; }); +els.media.addEventListener("error", () => notice("error", "MEDIA UNAVAILABLE", "The private evidence image could not be loaded. Refresh captures before retrying.")); +initialize(); diff --git a/tools/game-data-capture/src/game_data_capture/web/templates/index.html b/tools/game-data-capture/src/game_data_capture/web/templates/index.html new file mode 100644 index 0000000..6b274a1 --- /dev/null +++ b/tools/game-data-capture/src/game_data_capture/web/templates/index.html @@ -0,0 +1,126 @@ + + + + + + + Local Capture Review + + + + + +
+
+
+

LOCAL OPERATIONS / CAPTURE REVIEW

+

Evidence intake

+
+
+ LOCKED + Queue unavailable +
+
+ +
+ STARTUP + Checking this local session. +
+ +
+
+

CAPTURE SESSION

+

DISARMED

+
+
+ + + +
+
+ +
+ + +
+
+
+

MANUAL REVIEW

+

No capture selected

+
+ LOCKED +
+ +
+

Select a capture after the local session is unlocked. Media remains private to this browser session.

+
+ + +
+
+
+ + +
+

CONFIRMATION REQUIRED

+

Confirm action

+

+
+ + +
+
+ + + diff --git a/tools/game-data-capture/tests/contract/test_contracts.py b/tools/game-data-capture/tests/contract/test_contracts.py new file mode 100644 index 0000000..f09bed8 --- /dev/null +++ b/tools/game-data-capture/tests/contract/test_contracts.py @@ -0,0 +1,505 @@ +from __future__ import annotations + +import copy +import json +import math +from pathlib import Path +from typing import Any + +import pytest +from jsonschema import Draft202012Validator, FormatChecker +from jsonschema.exceptions import ValidationError +from jsonschema.protocols import Validator + +from game_data_capture.domain.ports import ResourceBudget +from game_data_capture.domain.state_machines import ( + STATE_MACHINES, + InvalidStateTransition, + require_transition, +) +from game_data_capture.domain.taxonomy import RECORD_TYPES_BY_VIEW, SCREEN_VIEWS, Family +from game_data_capture.domain.validation import ( + ContractValidationError, + validate_archive_document, + validate_classification, + validate_handoff_document, +) + +ROOT = Path(__file__).resolve().parents[2] +CONTRACTS = ROOT / "contracts" +CLASSIFICATION_EXAMPLES = CONTRACTS / "examples" / "classification" + + +def load_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as handle: + value = json.load(handle) + assert isinstance(value, dict) + return value + + +def schema_validator(schema_name: str) -> Validator: + schema = load_json(CONTRACTS / schema_name) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema, format_checker=FormatChecker()) + + +@pytest.mark.parametrize( + "schema_name", + [ + "screenshot-classification.schema.json", + "evidence-archive-manifest-v1.schema.json", + "isac-handoff-v1.schema.json", + ], +) +def test_schemas_compile_as_portable_draft_2020_12(schema_name: str) -> None: + schema = load_json(CONTRACTS / schema_name) + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert not any(key.startswith("x-python") for key in _all_keys(schema)) + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + assert validator.schema == schema + + +def _all_keys(value: object) -> list[str]: + keys: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + keys.append(key) + keys.extend(_all_keys(child)) + elif isinstance(value, list): + for child in value: + keys.extend(_all_keys(child)) + return keys + + +def test_domain_contract_and_python_taxonomy_cover_every_family_view() -> None: + contract = load_json(CONTRACTS / "domain-contract-v1.json") + families = contract["families"] + for family, views in SCREEN_VIEWS.items(): + assert set(families[family.value]["views"]) == set(views) + for view in views: + expected = [record_type.value for record_type in RECORD_TYPES_BY_VIEW[(family, view)]] + assert families[family.value]["views"][view] == expected + + +def test_domain_contract_and_python_state_machines_match() -> None: + contract_states = load_json(CONTRACTS / "domain-contract-v1.json")["states"] + assert set(contract_states) == set(STATE_MACHINES) + for machine_name, machine in STATE_MACHINES.items(): + expected = {state: sorted(targets) for state, targets in machine.transitions.items()} + actual = { + state: sorted(targets) for state, targets in contract_states[machine_name].items() + } + assert actual == expected + + +@pytest.mark.parametrize("family", [family.value for family in Family]) +def test_each_family_has_a_valid_and_invalid_example(family: str) -> None: + validator = schema_validator("screenshot-classification.schema.json") + valid = load_json(CLASSIFICATION_EXAMPLES / f"{family}.valid.json") + invalid = load_json(CLASSIFICATION_EXAMPLES / f"{family}.invalid.json") + validator.validate(valid) + with pytest.raises(ValidationError): + validator.validate(invalid) + validate_classification(valid) + with pytest.raises(ContractValidationError): + validate_classification(invalid) + + +def test_composite_classification_preserves_multiple_record_types() -> None: + classification = load_json(CLASSIFICATION_EXAMPLES / "weapon.valid.json") + validate_classification(classification) + assert classification["record_types"] == ["weapon", "weapon_talent"] + + +def test_unknown_enum_value_is_rejected() -> None: + classification = load_json(CLASSIFICATION_EXAMPLES / "gear.valid.json") + classification["gear_slot"] = "Vest" + with pytest.raises(ValidationError): + schema_validator("screenshot-classification.schema.json").validate(classification) + with pytest.raises(ContractValidationError, match="Gear slot"): + validate_classification(classification) + + +@pytest.mark.parametrize( + ("schema_name", "valid_fixture", "invalid_fixture"), + [ + ( + "evidence-archive-manifest-v1.schema.json", + "archive/manifest.valid.json", + "archive/manifest.invalid.json", + ), + ( + "isac-handoff-v1.schema.json", + "handoff/isac-handoff.valid.json", + "handoff/isac-handoff.invalid.json", + ), + ], +) +def test_versioned_schema_fixtures_validate_and_reject( + schema_name: str, valid_fixture: str, invalid_fixture: str +) -> None: + validator = schema_validator(schema_name) + validator.validate(load_json(CONTRACTS / "examples" / valid_fixture)) + with pytest.raises(ValidationError): + validator.validate(load_json(CONTRACTS / "examples" / invalid_fixture)) + + +@pytest.mark.parametrize( + ("schema_name", "valid_fixture"), + [ + ("evidence-archive-manifest-v1.schema.json", "archive/manifest.valid.json"), + ("isac-handoff-v1.schema.json", "handoff/isac-handoff.valid.json"), + ], +) +def test_versioned_schemas_reject_future_versions_from_otherwise_valid_documents( + schema_name: str, valid_fixture: str +) -> None: + document = load_json(CONTRACTS / "examples" / valid_fixture) + document["schema_version"] = 2 + with pytest.raises(ValidationError): + schema_validator(schema_name).validate(document) + + +def test_valid_archive_preserves_observed_values_without_canonicalizing_them() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + schema_validator("evidence-archive-manifest-v1.schema.json").validate(archive) + validate_archive_document(archive) + observed_paths = {field["path"] for field in archive["observed_instances"][0]["fields"]} + canonical_paths = {field["path"] for field in archive["canonical_candidates"][0]["fields"]} + assert "observed.damage" in observed_paths + assert "observed.damage" not in canonical_paths + assert archive["observed_instances"][0]["observed_context"]["upgrade_state"] == { + "path": "observed.upgrade_state", + "value": None, + "missing_reason": "not_visible", + "evidence_refs": ["evidence-upgrade"], + "uncertainty_note": "The screen does not show an upgrade indicator.", + } + + +def test_malformed_evidence_bounds_are_rejected() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["evidence_values"][0]["region"] = { + "coordinate_space": "normalized", + "x": 0.8, + "y": 0.1, + "width": 0.3, + "height": 0.1, + } + with pytest.raises(ContractValidationError, match="bounds"): + validate_archive_document(archive) + + +def test_archive_schema_rejects_nonpositive_evidence_bounds() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["evidence_values"][0]["region"]["width"] = 0 + with pytest.raises(ValidationError): + schema_validator("evidence-archive-manifest-v1.schema.json").validate(archive) + + +def test_null_evidence_without_missing_reason_is_rejected() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + missing = next( + value for value in archive["evidence_values"] if value["evidence_id"] == "evidence-upgrade" + ) + missing["missing_reason"] = None + with pytest.raises(ValidationError): + schema_validator("evidence-archive-manifest-v1.schema.json").validate(archive) + with pytest.raises(ContractValidationError, match="missing_reason"): + validate_archive_document(archive) + + +def test_archive_video_limit_matches_the_configured_16_gib_maximum() -> None: + validator = schema_validator("evidence-archive-manifest-v1.schema.json") + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + video = { + "artifact_id": "video-limit", + "sha256": "c" * 64, + "media_type": "video/mp4", + "byte_size": 16 * 1024 * 1024 * 1024, + "sensitivity": "private", + "parent_artifact_id": None, + "video_timestamp_ms": None, + } + archive["artifacts"].append(video) + validator.validate(archive) + video_byte_size = video["byte_size"] + assert isinstance(video_byte_size, int) + oversized_video_bytes = video_byte_size + 1 + video["byte_size"] = oversized_video_bytes + with pytest.raises(ValidationError): + validator.validate(archive) + + +def test_archive_image_limit_is_64_mib() -> None: + validator = schema_validator("evidence-archive-manifest-v1.schema.json") + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + artifact = archive["artifacts"][0] + artifact["byte_size"] = 64 * 1024 * 1024 + validator.validate(archive) + artifact["byte_size"] += 1 + with pytest.raises(ValidationError): + validator.validate(archive) + + +def test_image_handoff_limit_is_64_mib() -> None: + validator = schema_validator("isac-handoff-v1.schema.json") + handoff = load_json(CONTRACTS / "examples" / "handoff" / "isac-handoff.valid.json") + artifact = handoff["source_artifacts"][0] + artifact["byte_size"] = 64 * 1024 * 1024 + validator.validate(handoff) + artifact["byte_size"] += 1 + with pytest.raises(ValidationError): + validator.validate(handoff) + + +def test_dangling_evidence_reference_is_rejected() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["canonical_candidates"][0]["fields"][0]["evidence_refs"] = ["not-present"] + with pytest.raises(ContractValidationError, match="dangling references"): + validate_archive_document(archive) + + +@pytest.mark.parametrize( + ("collection", "id_field", "new_item"), + [ + ("artifacts", "artifact_id", None), + ("captures", "capture_id", None), + ("screens", "screen_id", None), + ("jobs", "job_id", {"job_id": "duplicate", "last_error_id": None}), + ( + "extractions", + "extraction_id", + { + "extraction_id": "duplicate", + "screen_id": "screen-001", + "evidence_refs": [], + }, + ), + ( + "groups", + "group_id", + { + "group_id": "duplicate", + "screen_ids": ["screen-001"], + "supersedes_group_ids": [], + }, + ), + ("reviews", "review_id", None), + ("exports", "export_id", {"export_id": "duplicate", "artifact_id": None}), + ("errors", "error_id", {"error_id": "duplicate"}), + ("evidence_values", "evidence_id", None), + ("observed_instances", "instance_id", None), + ("canonical_candidates", "candidate_id", None), + ], +) +def test_archive_rejects_duplicate_entity_ids( + collection: str, id_field: str, new_item: dict[str, Any] | None +) -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + item = new_item if new_item is not None else archive[collection][0] + archive[collection] = [copy.deepcopy(item), copy.deepcopy(item)] + with pytest.raises(ContractValidationError, match=f"Duplicate {id_field}"): + validate_archive_document(archive) + + +def test_archive_rejects_dangling_job_error_reference() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["jobs"].append({"job_id": "job-001", "last_error_id": "missing-error"}) + with pytest.raises(ContractValidationError, match=r"job\.last_error_id"): + validate_archive_document(archive) + + +@pytest.mark.parametrize("supersedes", [["missing-group"], ["group-001"]]) +def test_archive_rejects_invalid_group_supersession(supersedes: list[str]) -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["groups"].append( + { + "group_id": "group-001", + "screen_ids": ["screen-001"], + "supersedes_group_ids": supersedes, + } + ) + with pytest.raises(ContractValidationError, match="supersedes_group_ids"): + validate_archive_document(archive) + + +def test_archive_rejects_dangling_export_artifact_reference() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["exports"].append({"export_id": "export-001", "artifact_id": "missing-artifact"}) + with pytest.raises(ContractValidationError, match=r"export\.artifact_id"): + validate_archive_document(archive) + + +def test_archive_rejects_self_referential_artifact_parent() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + archive["artifacts"][0]["parent_artifact_id"] = "image-001" + with pytest.raises(ContractValidationError, match="cannot reference itself"): + validate_archive_document(archive) + + +def test_correction_history_final_state_must_match_current_evidence() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + evidence = next( + value for value in archive["evidence_values"] if value["evidence_id"] == "evidence-damage" + ) + evidence["normalized_value"] = 12401 + with pytest.raises(ContractValidationError, match="final state"): + validate_archive_document(archive) + + +def test_correction_history_rejects_disconnected_value_states() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + evidence = next( + value for value in archive["evidence_values"] if value["evidence_id"] == "evidence-damage" + ) + evidence["correction_history"].append( + { + "correction_id": "correction-002", + "corrected_at": "2026-07-31T12:01:00Z", + "actor": "local_user", + "reason": "Disconnected fixture.", + "before": {"raw_text": "wrong", "normalized_value": 0, "missing_reason": None}, + "after": { + "raw_text": evidence["raw_text"], + "normalized_value": evidence["normalized_value"], + "missing_reason": evidence["missing_reason"], + }, + } + ) + with pytest.raises(ContractValidationError, match="disconnected"): + validate_archive_document(archive) + + +def test_correction_history_rejects_out_of_order_timestamps() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + evidence = next( + value for value in archive["evidence_values"] if value["evidence_id"] == "evidence-damage" + ) + current_state = { + "raw_text": evidence["raw_text"], + "normalized_value": evidence["normalized_value"], + "missing_reason": evidence["missing_reason"], + } + evidence["correction_history"].append( + { + "correction_id": "correction-002", + "corrected_at": "2026-07-31T11:59:00Z", + "actor": "local_user", + "reason": "Out-of-order fixture.", + "before": copy.deepcopy(current_state), + "after": copy.deepcopy(current_state), + } + ) + with pytest.raises(ContractValidationError, match="strictly ordered"): + validate_archive_document(archive) + + +def test_correction_ids_are_unique_across_the_archive() -> None: + archive = load_json(CONTRACTS / "examples" / "archive" / "manifest.valid.json") + evidence = next( + value for value in archive["evidence_values"] if value["evidence_id"] == "evidence-damage" + ) + duplicate = copy.deepcopy(evidence) + duplicate["evidence_id"] = "evidence-damage-copy" + archive["evidence_values"].append(duplicate) + with pytest.raises(ContractValidationError, match="Duplicate correction_id"): + validate_archive_document(archive) + + +@pytest.mark.parametrize( + ("example", "validator"), + [ + ("archive/manifest.invalid.json", validate_archive_document), + ("handoff/isac-handoff.invalid.json", validate_handoff_document), + ], +) +def test_future_schema_versions_fail_closed(example: str, validator: Any) -> None: + with pytest.raises(ContractValidationError, match="Unsupported schema_version"): + validator(load_json(CONTRACTS / "examples" / example)) + + +def test_valid_handoff_has_no_mutation_authority_and_resolves_evidence() -> None: + handoff = load_json(CONTRACTS / "examples" / "handoff" / "isac-handoff.valid.json") + schema_validator("isac-handoff-v1.schema.json").validate(handoff) + validate_handoff_document(handoff) + assert handoff["trust_boundary"] == { + "intended_destination": "authenticated_isac_intake", + "mutation_authority": "none", + "requires_authenticated_action": True, + "preserves_moderation": True, + "creates_audit_events": False, + } + + +def test_handoff_cannot_claim_mutation_authority() -> None: + handoff = load_json(CONTRACTS / "examples" / "handoff" / "isac-handoff.valid.json") + handoff["trust_boundary"]["mutation_authority"] = "canonical_write" + with pytest.raises(ValidationError): + schema_validator("isac-handoff-v1.schema.json").validate(handoff) + with pytest.raises(ContractValidationError, match="trust boundary"): + validate_handoff_document(handoff) + + +def test_handoff_dangling_evidence_reference_is_rejected() -> None: + handoff = load_json(CONTRACTS / "examples" / "handoff" / "isac-handoff.valid.json") + broken = copy.deepcopy(handoff) + broken["records"][0]["fields"][0]["evidence_refs"] = ["missing"] + with pytest.raises(ContractValidationError, match="dangling references"): + validate_handoff_document(broken) + + +def test_handoff_source_archive_provenance_does_not_require_local_entities() -> None: + handoff = load_json(CONTRACTS / "examples" / "handoff" / "isac-handoff.valid.json") + handoff["records"][0]["source_candidate_ids"] = ["archive-candidate-not-embedded"] + handoff["records"][0]["observed_instance_ids"] = ["archive-instance-not-embedded"] + validate_handoff_document(handoff) + + +@pytest.mark.parametrize("field", ["max_input_bytes", "max_output_bytes", "max_items"]) +def test_resource_budget_rejects_negative_limits(field: str) -> None: + with pytest.raises(ValueError, match="non-negative integer"): + ResourceBudget( + deadline_monotonic=1.0, + max_input_bytes=-1 if field == "max_input_bytes" else 1, + max_output_bytes=-1 if field == "max_output_bytes" else 1, + max_items=-1 if field == "max_items" else 1, + ) + + +@pytest.mark.parametrize("deadline", [math.nan, math.inf, -math.inf]) +def test_resource_budget_rejects_nonfinite_deadlines(deadline: float) -> None: + with pytest.raises(ValueError, match="finite or null"): + ResourceBudget( + deadline_monotonic=deadline, + max_input_bytes=1, + max_output_bytes=1, + max_items=1, + ) + + +def test_resource_budget_allows_zero_limits_and_no_deadline() -> None: + assert ( + ResourceBudget( + deadline_monotonic=None, + max_input_bytes=0, + max_output_bytes=0, + max_items=0, + ).deadline_monotonic + is None + ) + + +@pytest.mark.parametrize("machine", sorted(STATE_MACHINES)) +def test_each_state_machine_rejects_an_invalid_transition(machine: str) -> None: + first_state = next(iter(STATE_MACHINES[machine].transitions)) + with pytest.raises(InvalidStateTransition): + require_transition(machine, first_state, "not-a-state") + + +def test_terminal_transition_is_rejected_and_recovery_transition_is_allowed() -> None: + with pytest.raises(InvalidStateTransition, match="Invalid review transition"): + require_transition("review", "cancelled", "approved") + require_transition("job", "failed", "queued") + require_transition("error", "retrying", "recovered") diff --git a/tools/game-data-capture/tests/integration/storage/test_corpus.py b/tools/game-data-capture/tests/integration/storage/test_corpus.py new file mode 100644 index 0000000..c98bcba --- /dev/null +++ b/tools/game-data-capture/tests/integration/storage/test_corpus.py @@ -0,0 +1,458 @@ +from __future__ import annotations + +import sqlite3 +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from game_data_capture.storage import CaptureCorpus, CaptureRepository, ContentAddressedBlobStore +from game_data_capture.storage.blobs import BlobCorruptionError, BlobLockError +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import StorageError + + +def _label(*, view: str = "detail", records: list[str] | None = None) -> dict[str, object]: + return { + "family": "weapon", + "view": view, + "gear_slot": None, + "record_types": ["weapon"] if records is None else records, + "confidence": 1.0, + "abstention_reason": None, + } + + +@pytest.fixture +def corpus(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository, CaptureCorpus]: + root = (tmp_path / "utility").resolve() + database = StorageDatabase( + root, + (Path(__file__).parents[3] / "migrations").resolve(), + ) + database.migrate() + blobs = ContentAddressedBlobStore(root) + repository = CaptureRepository(database, blobs) + return database, repository, CaptureCorpus(database, blobs, max_media_bytes=1024) + + +def _receive( + repository: CaptureRepository, capture_id: str, token: str, payload: bytes = b"png" +) -> None: + repository.receive(capture_id, token, [payload], media_type="image/png", max_bytes=1024) + + +def _assert_media_unavailable(port: CaptureCorpus, capture_id: str) -> None: + media = port.open_media(capture_id) + assert media is not None + with pytest.raises(BlobCorruptionError): + next(iter(media.chunks)) + + +def _commit_deletion_intent(database: StorageDatabase, capture_id: str) -> None: + with database.connect() as connection: + row = connection.execute( + "SELECT c.blob_sha256, c.state, c.received_at, c.receipt_token, c.failure_reason, " + "c.revision, b.byte_size FROM captures AS c JOIN blobs AS b " + "ON b.sha256 = c.blob_sha256 WHERE c.capture_id = ?", + (capture_id,), + ).fetchone() + assert row is not None + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO capture_deletion_intents(blob_sha256, capture_id, byte_size, " + "capture_state, received_at, receipt_token, failure_reason, revision, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + row["blob_sha256"], + capture_id, + row["byte_size"], + row["state"], + row["received_at"], + row["receipt_token"], + row["failure_reason"], + row["revision"], + "time", + ), + ) + connection.execute("DELETE FROM captures WHERE capture_id = ?", (capture_id,)) + connection.execute("COMMIT") + + +def test_safe_bounded_listing_detail_and_verified_media( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + _, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + _receive(repository, "capture-0002", "receipt-0002") + records = port.list_captures(limit=1) + assert len(records) == 1 and records[0].capture_id == "capture-0002" + assert port.get_capture("../capture-0001") is None + assert port.open_media("../capture-0001") is None + media = port.open_media("capture-0001") + assert media is not None + assert media.media_type == "image/png" and media.byte_size == 3 + chunks = media.chunks + assert next(iter(chunks)) == b"png" + close = getattr(chunks, "close", None) + assert callable(close) + close() + for invalid in (0, 257, True): + with pytest.raises(ValueError): + port.list_captures(limit=invalid) + + +def test_labels_preserve_prior_values_across_restart_and_reject_is_idempotent( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + first = _label() + second = _label(view="blueprint", records=["weapon_talent"]) + with pytest.raises(ValueError): + port.label("capture-0001", {"family": "weapon"}) + assert port.label("capture-0001", first) == "updated" + assert port.label("capture-0001", first) == "updated" + assert port.label("capture-0001", second) == "updated" + restarted = CaptureCorpus( + database, ContentAddressedBlobStore(database.root), max_media_bytes=1024 + ) + record = restarted.get_capture("capture-0001") + assert record is not None and record.classification == second + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1 + assert restarted.reject_preserving_evidence("capture-0001") == "updated" + assert restarted.reject_preserving_evidence("capture-0001") == "updated" + assert restarted.label("capture-0001", first) == "conflict" + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM capture_rejections").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1 + + +def test_concurrent_label_writes_serialize_without_losing_prior_value( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + labels = (_label(), _label(view="blueprint", records=["weapon_talent"])) + with ThreadPoolExecutor(max_workers=2) as executor: + outcomes = list(executor.map(lambda value: port.label("capture-0001", value), labels)) + assert outcomes == ["updated", "updated"] + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 1 + + +def test_canonical_equivalent_labels_do_not_append_history( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + first = _label(records=["weapon", "weapon_talent"]) + equivalent = _label(records=["weapon_talent", "weapon"]) + equivalent["confidence"] = 1 + assert port.label("capture-0001", first) == "updated" + assert port.label("capture-0001", equivalent) == "updated" + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0 + + +def test_rejected_or_evidence_bearing_captures_are_retained( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + _, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + assert port.reject_preserving_evidence("capture-0001") == "updated" + assert port.delete_with_retention("capture-0001") == "retained" + _receive(repository, "capture-0002", "receipt-0002") + repository.append_ocr_run("run-0002", "capture-0002", "ocr", "1", {}) + assert port.delete_with_retention("capture-0002") == "retained" + assert port.delete_with_retention("bad") == "not_found" + + +def test_label_and_job_evidence_retain_capture( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + assert port.label("capture-0001", _label()) == "updated" + assert port.delete_with_retention("capture-0001") == "retained" + _receive(repository, "capture-0002", "receipt-0002") + with database.connect() as connection: + connection.execute( + "INSERT INTO jobs(job_id, capture_id, kind, state, max_attempts) " + "VALUES (?, ?, ?, ?, ?)", + ("job-0002", "capture-0002", "ocr", "queued", 1), + ) + assert port.delete_with_retention("capture-0002") == "retained" + + +def test_receive_translates_put_lock_failure_without_capture( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], + monkeypatch: pytest.MonkeyPatch, +) -> None: + database, repository, _ = corpus + + def locked(_: object, *, max_bytes: int) -> tuple[str, int]: + raise BlobLockError("locked") + + monkeypatch.setattr(repository.blobs, "put", locked) + with pytest.raises(StorageError, match="unavailable"): + _receive(repository, "capture-0001", "receipt-0001") + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0 + + +@pytest.mark.parametrize( + ("state", "failure_reason"), + (("failed", None), ("cancelled", None), ("stored", "capture failure"), ("unknown", None)), +) +def test_non_disposable_capture_rows_are_retained( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], + state: str, + failure_reason: str | None, +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + with database.connect() as connection: + connection.execute( + "UPDATE captures SET state = ?, failure_reason = ? WHERE capture_id = ?", + (state, failure_reason, "capture-0001"), + ) + assert port.delete_with_retention("capture-0001") == "retained" + + +def test_capture_lineage_and_failure_evidence_are_retained( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + repository.append_lineage("lineage-0001", "capture-0001", "parent-0001", "derived") + assert port.delete_with_retention("capture-0001") == "retained" + _receive(repository, "capture-0002", "receipt-0002") + repository.record_failure("failure-0002", "capture", "capture-0002", "fault", "bounded") + assert port.delete_with_retention("capture-0002") == "retained" + _receive(repository, "capture-0003", "receipt-0003") + repository.append_lineage("lineage-0003", "child-0003", "capture-0003", "derived") + assert port.delete_with_retention("capture-0003") == "retained" + + +def test_shared_blob_deletion_keeps_remaining_capture_media( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + _, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001", b"shared") + _receive(repository, "capture-0002", "receipt-0002", b"shared") + assert port.delete_with_retention("capture-0001") == "updated" + assert port.get_capture("capture-0001") is None + media = port.open_media("capture-0002") + assert media is not None and b"".join(media.chunks) == b"shared" + with repository.database.connect() as connection: + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + + +def test_unshared_deletion_removes_media_metadata_and_intent( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone() + assert row is not None + path = repository.blobs._path(str(row["blob_sha256"])) + assert port.delete_with_retention("capture-0001") == "updated" + assert not path.exists() + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0 + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0 + ) + + +def test_committed_intent_before_unlink_blocks_replay_then_finalizes( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone() + assert row is not None + path = repository.blobs._path(str(row["blob_sha256"])) + _commit_deletion_intent(database, "capture-0001") + assert path.exists() + with pytest.raises(StorageError, match="deletion"): + _receive(repository, "capture-0002", "receipt-0001") + restarted = CaptureCorpus( + database, ContentAddressedBlobStore(database.root), max_media_bytes=1024 + ) + assert restarted.reconcile_deletions() == 1 + assert not path.exists() + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0 + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0 + ) + + +def test_interleaved_reinstall_waits_for_deletion_finalization( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], + monkeypatch: pytest.MonkeyPatch, +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001", b"shared") + unlinked = threading.Event() + release = threading.Event() + original = repository.blobs.delete_verified + outcomes: list[object] = [] + + def pause(digest: str, *, expected_size: int) -> bool: + result = original(digest, expected_size=expected_size) + unlinked.set() + assert release.wait(0.5) + return result + + monkeypatch.setattr(repository.blobs, "delete_verified", pause) + deletion = threading.Thread( + target=lambda: outcomes.append(port.delete_with_retention("capture-0001")) + ) + deletion.start() + assert unlinked.wait(0.5) + receipt = threading.Thread( + target=lambda: outcomes.append( + repository.receive( + "capture-0002", "receipt-0002", [b"shared"], media_type="image/png", max_bytes=1024 + ) + ) + ) + receipt.start() + release.set() + deletion.join(1) + receipt.join(1) + assert not deletion.is_alive() and not receipt.is_alive() + assert set(outcomes) == {"updated", "capture-0002"} + with database.connect() as connection: + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0 + ) + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + media = port.open_media("capture-0002") + assert media is not None and b"".join(media.chunks) == b"shared" + + +def test_unlink_then_sqlite_finalization_failure_recovers( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + row = database.connect().execute("SELECT sha256, byte_size FROM blobs").fetchone() + assert row is not None + path = repository.blobs._path(str(row["sha256"])) + _commit_deletion_intent(database, "capture-0001") + repository.blobs.delete_verified(str(row["sha256"]), expected_size=int(row["byte_size"])) + assert not path.exists() + with database.connect() as connection: + connection.execute( + "CREATE TRIGGER fail_finalization BEFORE DELETE ON capture_deletion_intents " + "BEGIN SELECT RAISE(ABORT, 'fault'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="fault"): + port.reconcile_deletions() + assert port.get_capture("capture-0001") is None + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 1 + ) + connection.execute("DROP TRIGGER fail_finalization") + restarted = CaptureCorpus( + database, ContentAddressedBlobStore(database.root), max_media_bytes=1024 + ) + assert restarted.reconcile_deletions() == 1 + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 0 + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0 + ) + assert not path.exists() + + +def test_corrupt_missing_and_symlink_media_fail_closed( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], tmp_path: Path +) -> None: + _, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + with repository.database.connect() as connection: + row = connection.execute("SELECT blob_sha256 FROM captures").fetchone() + assert row is not None + path = repository.blobs._path(str(row["blob_sha256"])) + path.write_bytes(b"x" * 2048) + _assert_media_unavailable(port, "capture-0001") + path.unlink() + _assert_media_unavailable(port, "capture-0001") + path.symlink_to(tmp_path / "outside") + _assert_media_unavailable(port, "capture-0001") + + +def test_label_transaction_rolls_back_on_evidence_failure( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + assert port.label("capture-0001", _label()) == "updated" + with database.connect() as connection: + connection.execute( + "CREATE TRIGGER fail_label_evidence BEFORE INSERT ON capture_label_evidence " + "BEGIN SELECT RAISE(ABORT, 'fault'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="fault"): + port.label( + "capture-0001", + _label(view="blueprint", records=["weapon_talent"]), + ) + assert port.get_capture("capture-0001") is not None + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM capture_label_evidence").fetchone()[0] == 0 + stored = connection.execute("SELECT classification_json FROM capture_labels").fetchone() + assert stored is not None and '"view":"detail"' in stored["classification_json"] + + +def test_deletion_intent_recovers_after_unlink_failure( + corpus: tuple[StorageDatabase, CaptureRepository, CaptureCorpus], + monkeypatch: pytest.MonkeyPatch, +) -> None: + database, repository, port = corpus + _receive(repository, "capture-0001", "receipt-0001") + with database.connect() as connection: + original = tuple( + connection.execute( + "SELECT capture_id, blob_sha256, state, received_at, receipt_token, " + "failure_reason, revision FROM captures" + ).fetchone() + ) + + def fail_unlink(_: str, *, expected_size: int) -> bool: + raise BlobCorruptionError("injected fault") + + monkeypatch.setattr(repository.blobs, "delete_verified", fail_unlink) + assert port.delete_with_retention("capture-0001") == "retained" + with database.connect() as connection: + assert ( + connection.execute("SELECT count(*) FROM capture_deletion_intents").fetchone()[0] == 0 + ) + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1 + assert ( + tuple( + connection.execute( + "SELECT capture_id, blob_sha256, state, received_at, receipt_token, " + "failure_reason, revision FROM captures" + ).fetchone() + ) + == original + ) + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + media = port.open_media("capture-0001") + assert media is not None and b"".join(media.chunks) == b"png" diff --git a/tools/game-data-capture/tests/integration/storage/test_database.py b/tools/game-data-capture/tests/integration/storage/test_database.py new file mode 100644 index 0000000..13b2ebd --- /dev/null +++ b/tools/game-data-capture/tests/integration/storage/test_database.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import shutil +import sqlite3 +from pathlib import Path + +import pytest + +from game_data_capture.storage.database import MigrationError, StorageDatabase + + +def _migrations(tmp_path: Path) -> Path: + source = Path(__file__).parents[3] / "migrations" / "001_initial.sql" + destination = tmp_path / "migrations" + destination.mkdir() + shutil.copy(source, destination / source.name) + return destination + + +def test_connection_pragmas_and_private_modes(tmp_path: Path) -> None: + root = (tmp_path / "utility").resolve() + database = StorageDatabase(root, _migrations(tmp_path).resolve()) + database.migrate() + with database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute("CREATE TABLE IF NOT EXISTS live_wal(value INTEGER)") + assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 + assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal" + assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000 + assert connection.execute("PRAGMA synchronous").fetchone()[0] == 2 + for path in ( + database.path.with_name("capture.sqlite3-wal"), + database.path.with_name("capture.sqlite3-shm"), + ): + if path.exists(): + assert path.stat().st_mode & 0o077 == 0 + connection.execute("COMMIT") + assert root.stat().st_mode & 0o777 == 0o700 + assert database.path.stat().st_mode & 0o777 == 0o600 + + +def test_database_symlink_is_rejected(tmp_path: Path) -> None: + root = (tmp_path / "utility").resolve() + root.mkdir() + outside = tmp_path / "outside" + outside.write_text("x") + (root / "capture.sqlite3").symlink_to(outside) + with pytest.raises(MigrationError): + StorageDatabase(root, _migrations(tmp_path).resolve()).connect() + + +def test_migration_history_and_failure_rollback(tmp_path: Path) -> None: + root = (tmp_path / "utility").resolve() + migrations = _migrations(tmp_path).resolve() + database = StorageDatabase(root, migrations) + database.migrate() + first = migrations / "001_initial.sql" + first.write_text(first.read_text() + "\n-- changed\n") + with pytest.raises(MigrationError): + database.migrate() + first.write_text((Path(__file__).parents[3] / "migrations" / "001_initial.sql").read_text()) + bad = migrations / "002_bad.sql" + bad.write_text("CREATE TABLE partial_test(value INTEGER); BAD SQL;") + with pytest.raises(sqlite3.OperationalError): + database.migrate() + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 1 + assert ( + connection.execute( + "SELECT name FROM sqlite_master WHERE name='partial_test'" + ).fetchone() + is None + ) + bad.write_text("CREATE TABLE partial_test(value INTEGER);") + database.migrate() + database.migrate() + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2 + + +def test_migration_removal_rename_and_duplicate_rejected(tmp_path: Path) -> None: + root = (tmp_path / "utility").resolve() + migrations = _migrations(tmp_path).resolve() + database = StorageDatabase(root, migrations) + database.migrate() + path = migrations / "001_initial.sql" + path.rename(migrations / "001_renamed.sql") + with pytest.raises(MigrationError): + database.migrate() + (migrations / "001_renamed.sql").rename(path) + path.unlink() + with pytest.raises(MigrationError): + database.migrate() + shutil.copy(Path(__file__).parents[3] / "migrations" / "001_initial.sql", path) + shutil.copy(path, migrations / "001_duplicate.sql") + with pytest.raises(MigrationError): + database.migrate() + + +def test_corpus_migration_preserves_populated_initial_database(tmp_path: Path) -> None: + root = (tmp_path / "utility").resolve() + migrations = _migrations(tmp_path).resolve() + database = StorageDatabase(root, migrations) + database.migrate() + with database.connect() as connection: + connection.execute( + "INSERT INTO blobs(sha256, byte_size, media_type, created_at) VALUES (?, 1, ?, ?)", + ("0" * 64, "image/png", "time"), + ) + connection.execute( + "INSERT INTO captures(capture_id, blob_sha256, state, received_at, receipt_token) " + "VALUES (?, ?, ?, ?, ?)", + ("capture-0001", "0" * 64, "stored", "time", "receipt-0001"), + ) + source = Path(__file__).parents[3] / "migrations" / "002_capture_corpus.sql" + shutil.copy(source, migrations / source.name) + database.migrate() + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM capture_labels").fetchone()[0] == 0 + assert tuple( + connection.execute( + "SELECT sha256, byte_size, media_type, created_at FROM blobs" + ).fetchone() + ) == ("0" * 64, 1, "image/png", "time") + assert tuple( + connection.execute( + "SELECT capture_id, blob_sha256, state, received_at, receipt_token, " + "failure_reason, revision FROM captures" + ).fetchone() + ) == ("capture-0001", "0" * 64, "stored", "time", "receipt-0001", None, 0) + assert connection.execute("SELECT count(*) FROM schema_migrations").fetchone()[0] == 2 + migration = connection.execute( + "SELECT filename, checksum FROM schema_migrations WHERE version = 2" + ).fetchone() + assert migration is not None + assert migration["filename"] == "002_capture_corpus.sql" + assert isinstance(migration["checksum"], str) and len(migration["checksum"]) == 64 + names = {row["name"] for row in connection.execute("SELECT name FROM sqlite_master")} + assert { + "captures_blob_sha256", + "jobs_capture_id", + "ocr_runs_capture_id", + "runtime_failures_capture_subject", + "lineage_child_id", + "lineage_parent_id", + }.issubset(names) diff --git a/tools/game-data-capture/tests/integration/storage/test_jobs.py b/tools/game-data-capture/tests/integration/storage/test_jobs.py new file mode 100644 index 0000000..d3e715e --- /dev/null +++ b/tools/game-data-capture/tests/integration/storage/test_jobs.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from game_data_capture.pipeline.jobs import JobLeases +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import StorageError + + +@pytest.fixture +def database(tmp_path: Path) -> StorageDatabase: + database = StorageDatabase( + (tmp_path / "utility").resolve(), (Path(__file__).parents[3] / "migrations").resolve() + ) + database.migrate() + return database + + +def _expire(database: StorageDatabase, job_id: str) -> None: + with database.connect() as connection: + connection.execute( + "UPDATE jobs SET lease_expires_at = ? WHERE job_id = ?", + ((datetime.now(UTC) - timedelta(seconds=1)).isoformat(), job_id), + ) + + +def test_job_validation_and_exclusive_ownership(database: StorageDatabase) -> None: + jobs = JobLeases(database) + for job_id, kind, attempts in ( + ("!", "ocr", 1), + ("job", "bad", 1), + ("job", "ocr", True), + ("job", "ocr", 0), + ): + with pytest.raises(ValueError): + jobs.create(job_id, kind, max_attempts=attempts) + jobs.create("job-1", "ocr") + for duration in (0, 301, True): + with pytest.raises(ValueError): + jobs.acquire("owner-1", duration) + with pytest.raises(ValueError): + jobs.acquire("!", 30) + assert jobs.acquire("owner-1", 30) == "job-1" + assert jobs.acquire("owner-2", 30) is None + with pytest.raises(StorageError): + jobs.renew("job-1", "owner-2", 30) + with pytest.raises(StorageError): + jobs.finish("job-1", "owner-2", "succeeded") + for duration in (0, 301, True): + with pytest.raises(ValueError): + jobs.renew("job-1", "owner-1", duration) + + +def test_expiry_recovery_exhaustion_and_terminal_jobs(database: StorageDatabase) -> None: + jobs = JobLeases(database) + jobs.create("retry", "ocr", max_attempts=2) + assert jobs.acquire("owner-1", 30) == "retry" + _expire(database, "retry") + with pytest.raises(StorageError): + jobs.renew("retry", "owner-1", 30) + with pytest.raises(StorageError): + jobs.finish("retry", "owner-1", "succeeded") + assert jobs.recover() == 1 + with database.connect() as connection: + assert tuple( + connection.execute( + "SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id='retry'" + ).fetchone() + ) == ("queued", None, None) + assert JobLeases(database).acquire("owner-2", 30) == "retry" + with database.connect() as connection: + assert tuple( + connection.execute( + "SELECT attempt, state, lease_owner FROM jobs WHERE job_id='retry'" + ).fetchone() + ) == (2, "running", "owner-2") + jobs.create("exhaust", "ocr", max_attempts=1) + assert jobs.acquire("owner-3", 30) == "exhaust" + _expire(database, "exhaust") + assert jobs.recover() == 1 + with database.connect() as connection: + assert tuple( + connection.execute( + "SELECT state, attempt, lease_owner, lease_expires_at " + "FROM jobs WHERE job_id='exhaust'" + ).fetchone() + ) == ("failed", 1, None, None) + assert jobs.acquire("owner-4", 30) is None + for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")): + jobs.create(job_id, "ocr") + assert jobs.acquire("owner-5", 30) == job_id + jobs.finish(job_id, "owner-5", target) + with database.connect() as connection: + for job_id, target in (("ok", "succeeded"), ("fail", "failed"), ("cancel", "cancelled")): + assert tuple( + connection.execute( + "SELECT state, lease_owner, lease_expires_at FROM jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + ) == (target, None, None) + + +def test_retry_wait_finish_rejected_without_changing_lease(database: StorageDatabase) -> None: + jobs = JobLeases(database) + jobs.create("job-1", "ocr") + jobs.acquire("owner-1", 30) + with pytest.raises(ValueError): + jobs.finish("job-1", "owner-1", "retry_wait") + with database.connect() as connection: + assert tuple( + connection.execute( + "SELECT state, lease_owner FROM jobs WHERE job_id='job-1'" + ).fetchone() + ) == ("running", "owner-1") diff --git a/tools/game-data-capture/tests/integration/storage/test_recovery.py b/tools/game-data-capture/tests/integration/storage/test_recovery.py new file mode 100644 index 0000000..fcf1d25 --- /dev/null +++ b/tools/game-data-capture/tests/integration/storage/test_recovery.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from game_data_capture.pipeline.jobs import JobLeases +from game_data_capture.storage.blobs import ContentAddressedBlobStore +from game_data_capture.storage.database import StorageDatabase +from game_data_capture.storage.repository import CaptureRepository, StorageError + + +@pytest.fixture +def storage(tmp_path: Path) -> tuple[StorageDatabase, CaptureRepository]: + root = (tmp_path / "utility").resolve() + migrations = Path(__file__).parents[3] / "migrations" + database = StorageDatabase(root, migrations.resolve()) + database.migrate() + return database, CaptureRepository(database, ContentAddressedBlobStore(root)) + + +def test_identical_occurrences_are_deduplicated_and_receipts_survive_restart( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + assert ( + repository.receive( + "capture-1", "receipt-1", [b"same"], media_type="image/png", max_bytes=100 + ) + == "capture-1" + ) + assert ( + repository.receive( + "capture-2", "receipt-2", [b"same"], media_type="image/png", max_bytes=100 + ) + == "capture-2" + ) + # A simulated restart uses fresh repository objects and a repeated intake receipt. + restarted = CaptureRepository(database, ContentAddressedBlobStore(database.root)) + assert ( + restarted.receive( + "capture-other", "receipt-2", [b"same"], media_type="image/png", max_bytes=100 + ) + == "capture-2" + ) + assert restarted.report()["blobs"] == 1 + assert restarted.report()["captures"] == 2 + + +def test_capture_never_references_missing_blob_and_cas_rejects_stale( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"complete"], media_type="image/png", max_bytes=100 + ) + row = database.connect().execute("SELECT blob_sha256 FROM captures").fetchone() + assert row is not None and repository.blobs.verify(str(row["blob_sha256"])) == len(b"complete") + assert repository.transition("capture-1", "queued", 0) == 1 + with pytest.raises(StorageError): + repository.transition("capture-1", "failed", 0) + + +def test_reprocessing_is_append_only_and_leases_enforce_owner( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100 + ) + repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {"raw": "one"}) + repository.append_ocr_run("run-2", "capture-1", "ocr", "2", {"raw": "two"}) + assert database.connect().execute("SELECT count(*) FROM ocr_runs").fetchone()[0] == 2 + jobs = JobLeases(database) + jobs.create("job-1", "ocr", "capture-1") + assert jobs.acquire("worker-a", 30) == "job-1" + with pytest.raises(StorageError): + jobs.finish("job-1", "worker-b", "succeeded") + jobs.finish("job-1", "worker-a", "succeeded") + + +def test_replayed_receipt_does_not_consume_payload( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100 + ) + + consumed = [False] + + class MustNotConsume: + def __iter__(self) -> Iterator[bytes]: + consumed[0] = True + raise AssertionError("receipt replay consumed payload") + + assert ( + repository.receive( + "capture-2", "receipt-1", MustNotConsume(), media_type="image/png", max_bytes=100 + ) + == "capture-1" + ) + assert not consumed[0] + + +def test_every_provenance_kind_is_insert_only( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100 + ) + repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {}) + repository.append_region("region-1", "run-1", {}) + repository.append_classification("class-1", "run-1", {}) + repository.append_candidate("candidate-1", "run-1", {}) + repository.append_routing("route-1", "candidate-1", {}) + repository.append_review("review-1", "candidate-1", {}) + repository.append_lineage("lineage-1", "child-1", "parent-1", "derived") + repository.append_export("export-1", "ready", {}) + with pytest.raises(sqlite3.IntegrityError): + repository.append_ocr_run("run-1", "capture-1", "ocr", "1", {}) + payloads = { + "ocr_runs": "payload_json", + "regions": "region_json", + "classifications": "classification_json", + "extraction_candidates": "candidate_json", + "routing_decisions": "decision_json", + "review_events": "event_json", + "exports": "manifest_json", + } + with database.connect() as connection: + for table in ( + "ocr_runs", + "regions", + "classifications", + "extraction_candidates", + "routing_decisions", + "review_events", + "lineage", + "exports", + ): + assert connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0] == 1 + for table, column in payloads.items(): + json.loads(connection.execute(f"SELECT {column} FROM {table}").fetchone()[0]) + assert ( + connection.execute("SELECT count(*) FROM ocr_runs WHERE run_id='run-1'").fetchone()[0] + == 1 + ) + + +def test_failure_sanitization_and_provenance_bounds( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"capture"], media_type="image/png", max_bytes=100 + ) + with pytest.raises(ValueError): + repository.append_ocr_run("run", "capture-1", "", "1", {}) + with pytest.raises(ValueError): + repository.append_ocr_run("run", "capture-1", "x" * 101, "1", {}) + with pytest.raises(ValueError): + repository.append_lineage("line", "child", "parent", "") + with pytest.raises(ValueError): + repository.append_export("export", "", {}) + with pytest.raises(ValueError): + repository.append_candidate("candidate", "run", "x" * 1_000_001) + long_token = "x" * 600 + short_message = ( + "sk-secret123 Authorization: Bearer auth-secret token=token-secret secret=hidden " + "password=pw-secret api_key=key-secret /home/agent/private.png\n" + ) + repository.record_failure("failure-1", "capture", "capture-1", "failure-code", short_message) + repository.record_failure("failure-2", "capture", "capture-1", "failure-code", long_token) + with database.connect() as connection: + rows = list( + connection.execute("SELECT code, message FROM runtime_failures ORDER BY failure_id") + ) + assert len(rows) == 2 + for row in rows: + assert len(row["message"]) <= 512 and len(row["code"]) <= 64 + assert "\n" not in row["message"] + for value in ( + "sk-secret123", + "auth-secret", + "token-secret", + "hidden", + "pw-secret", + "key-secret", + "/home/agent/private.png", + "\n", + ): + assert value not in rows[0]["message"] + assert long_token not in rows[1]["message"] + with pytest.raises(ValueError): + repository.record_failure("failure-2", "BAD", "capture-1", "code", "message") + + +def test_complete_blob_after_capture_insert_failure_is_purgeable_orphan( + storage: tuple[StorageDatabase, CaptureRepository], +) -> None: + database, repository = storage + repository.receive( + "capture-1", "receipt-1", [b"payload-a"], media_type="image/png", max_bytes=100 + ) + payload_b = b"payload-b" + digest_b = hashlib.sha256(payload_b).hexdigest() + with pytest.raises(sqlite3.IntegrityError): + repository.receive( + "capture-1", "receipt-2", [payload_b], media_type="image/png", max_bytes=100 + ) + assert repository.blobs.verify(digest_b) == len(payload_b) + with database.connect() as connection: + assert ( + connection.execute( + "SELECT count(*) FROM blobs WHERE sha256 = ?", (digest_b,) + ).fetchone()[0] + == 0 + ) + assert ( + connection.execute( + "SELECT count(*) FROM captures WHERE blob_sha256 = ?", (digest_b,) + ).fetchone()[0] + == 0 + ) + assert repository.blobs._path(digest_b).relative_to( + repository.blobs.root + ) in repository.blobs.purge_dry_run(set()) diff --git a/tools/game-data-capture/tests/integration/test_capture_labeling.py b/tools/game-data-capture/tests/integration/test_capture_labeling.py new file mode 100644 index 0000000..07eaac5 --- /dev/null +++ b/tools/game-data-capture/tests/integration/test_capture_labeling.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +from fastapi.testclient import TestClient + +from game_data_capture.config import ServerConfig +from game_data_capture.web.app import ( + CaptureRecord, + CaptureSessionStatus, + ControlOutcome, + MediaDescriptor, + MutationOutcome, + WebConfig, + create_app, +) +from game_data_capture.web.security import new_bootstrap_token + +_ID = "capture-00000000000000000000000000000002" + + +@dataclass +class CaptureControl: + armed: bool = False + + def status(self) -> CaptureSessionStatus: + return CaptureSessionStatus(self.armed, 1, 64) + + def start_session(self) -> ControlOutcome: + self.armed = True + return "started" + + def stop_session(self) -> ControlOutcome: + self.armed = False + return "stopped" + + +class Corpus: + def __init__(self) -> None: + self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", None) + self.rejected = False + self.deleted = False + + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + return (self.record,) + + def get_capture(self, capture_id: str) -> CaptureRecord | None: + return self.record if capture_id == _ID else None + + def open_media(self, capture_id: str) -> MediaDescriptor | None: + return MediaDescriptor("image/png", 3, [b"png"]) if capture_id == _ID else None + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: + if capture_id != _ID: + return "not_found" + self.record = CaptureRecord(_ID, "stored", "2026-07-31T00:00:00Z", dict(classification)) + return "updated" + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: + if capture_id != _ID: + return "not_found" + self.rejected = True + return "updated" + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: + if capture_id != _ID: + return "not_found" + self.deleted = True + return "updated" + + +def test_private_capture_labeling_flow_preserves_rejection_before_retention_delete() -> None: + token = new_bootstrap_token() + control = CaptureControl() + corpus = Corpus() + client = TestClient( + create_app( + config=WebConfig(ServerConfig()), + capture_control=control, + corpus=corpus, + bootstrap_token=token, + ), + base_url="http://127.0.0.1:8765", + ) + origin = "http://127.0.0.1:8765" + csrf = str( + client.post("/api/bootstrap", headers={"Origin": origin}, json={"token": token}).json()[ + "csrf_token" + ] + ) + label = { + "family": "gear", + "view": "detail", + "gear_slot": "Mask", + "record_types": ["gear", "brand_set"], + "confidence": 1.0, + "abstention_reason": None, + } + + response = client.post( + f"/api/captures/{_ID}/label", headers={"Origin": origin, "X-CSRF-Token": csrf}, json=label + ) + assert response.status_code == 200 + csrf = str(response.json()["csrf_token"]) + assert client.get(f"/api/captures/{_ID}").json()["capture"]["classification"] == label + response = client.post( + f"/api/captures/{_ID}/reject", + headers={"Origin": origin, "X-CSRF-Token": csrf}, + json={"confirmation": "REJECT"}, + ) + assert response.status_code == 200 + csrf = str(response.json()["csrf_token"]) + assert corpus.rejected is True + response = client.post( + f"/api/captures/{_ID}/delete", + headers={"Origin": origin, "X-CSRF-Token": csrf}, + json={"confirmation": "DELETE"}, + ) + assert response.status_code == 200 + assert corpus.deleted is True + + +def test_review_ui_uses_local_external_assets_and_keeps_guardrails_in_static_source() -> None: + web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web" + template = (web_root / "templates" / "index.html").read_text(encoding="utf-8") + script = (web_root / "static" / "review.js").read_text(encoding="utf-8") + stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8") + + assert 'href="/static/review.css"' in template + assert 'src="/static/review.js"' in template + assert "" not in template + assert "http://" not in template + script + stylesheet + assert "https://" not in template + script + stylesheet + assert "innerHTML" not in script + assert "Origin" not in script + assert ( + 'history.replaceState(null, "", `${window.location.pathname}${window.location.search}`)' + in script + ) + assert '"/api/session/start"' in script + assert "JSON.stringify({ confirmation })" in script + assert "confidence: 1.0" in script + assert 'credentials: "same-origin"' in script + assert "localStorage" not in script + assert "sessionStorage" not in script + for expected in ( + "weapon", + "gear", + "skill_chip", + "os_protocol", + "specialization", + "unknown", + "Mask", + "Gloves", + "Backpack", + "Body Armor", + "Holster", + "Knee Pads", + "prefers-reduced-motion", + ): + assert expected in script + stylesheet + + +def test_review_page_and_local_static_assets_serve_with_security_headers() -> None: + client = TestClient( + create_app( + config=WebConfig(ServerConfig()), + capture_control=CaptureControl(), + corpus=Corpus(), + bootstrap_token=new_bootstrap_token(), + ), + base_url="http://127.0.0.1:8765", + ) + + page = client.get("/") + stylesheet = client.get("/static/review.css") + script = client.get("/static/review.js") + + for response in (page, stylesheet, script): + assert response.status_code == 200 + assert response.headers["content-security-policy"].startswith("default-src 'self'") + assert response.headers["referrer-policy"] == "no-referrer" + assert response.headers["x-content-type-options"] == "nosniff" + assert response.headers["x-frame-options"] == "DENY" + assert "http://" not in response.text + assert "https://" not in response.text + assert 'href="/static/review.css"' in page.text + assert 'src="/static/review.js"' in page.text + assert stylesheet.headers["content-type"].startswith("text/css") + assert "csrfPattern" in script.text + assert "setCheckedMembers" in script.text + assert "announce: false" in script.text diff --git a/tools/game-data-capture/tests/integration/test_clipboard_watch.py b/tools/game-data-capture/tests/integration/test_clipboard_watch.py new file mode 100644 index 0000000..0e00085 --- /dev/null +++ b/tools/game-data-capture/tests/integration/test_clipboard_watch.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from game_data_capture.ingest.clipboard import ClipboardIntake, WatchConfig + + +def test_shutdown_reaps_real_child_process_group( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + child: subprocess.Popen[bytes] | None = None + real_popen = subprocess.Popen + + def fake_popen(args: list[str], **kwargs: object) -> subprocess.Popen[bytes]: + nonlocal child + assert args[0] == "wl-paste" and kwargs["shell"] is False + child = real_popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return child + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + intake = ClipboardIntake( + WatchConfig((tmp_path / "spool").resolve(), 100, 1, "v1", 1, 0.01, 0.2) + ) + intake.arm() + assert child is not None and child.poll() is None + intake.shutdown() + for _ in range(20): + if child.poll() is not None: + break + time.sleep(0.01) + assert child.poll() is not None diff --git a/tools/game-data-capture/tests/integration/test_review_ui.py b/tools/game-data-capture/tests/integration/test_review_ui.py new file mode 100644 index 0000000..d6822cb --- /dev/null +++ b/tools/game-data-capture/tests/integration/test_review_ui.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from pathlib import Path + + +def test_review_interface_marks_keyboard_modal_and_responsive_state_hooks() -> None: + web_root = Path(__file__).parents[2] / "src" / "game_data_capture" / "web" + template = (web_root / "templates" / "index.html").read_text(encoding="utf-8") + script = (web_root / "static" / "review.js").read_text(encoding="utf-8") + stylesheet = (web_root / "static" / "review.css").read_text(encoding="utf-8") + + assert 'aria-live="polite"' in template + assert ' None: + script = ( + Path(__file__).parents[2] / "src" / "game_data_capture" / "web" / "static" / "review.js" + ).read_text(encoding="utf-8") + + assert 'headers.set("Origin"' not in script + assert "Origin:" not in script + assert 'const bootstrap = await request("/api/bootstrap"' in script + assert 'const session = await request("/api/session")' in script + assert script.index('"/api/bootstrap"') < script.index('"/api/session"') + assert "if (!validSession(session))" in script + assert "family, false, () =>" in script + assert "view, false, renderDependentFields" in script + assert "if (!classification) { renderViews(); renderDependentFields(); return; }" in script + assert "function routeForAction(action)" in script + assert '(action === "reject" || action === "delete") && activeCapture' in script + assert "const routes =" not in script + assert 'document.querySelectorAll("#label-form input")' in script + assert "input.disabled = pending || locked || !activeCapture" in script + assert 'document.querySelectorAll("#capture-list button")' in script + assert "button.disabled = pending || locked;" in script + assert "armed === true" in script + assert "armed !== true" in script + assert "function acceptMutationCsrf(payload)" in script + assert "if (!payload || !acceptMutationCsrf(payload))" in script + assert 'retained ? "RETAINED"' in script + assert "function setCheckedMembers(name, values)" in script + assert "input.checked = selectedValues.has(input.value);" in script + assert 'setCheckedMembers("record_type", classification.record_types || []);' in script + assert 'forEach((record) => setChecked("record_type", record))' not in script + assert ( + "async function refreshCaptures({ announce = true, fallbackFocus = null } = {})" in script + ) + assert "async function selectCapture(captureId, { announce = true } = {})" in script + assert 'if (announce) notice("success", "QUEUE REFRESHED"' in script + assert 'if (announce) notice("success", "CAPTURE READY"' in script + assert "refreshCaptures({ announce: false })" in script + assert "selectCapture(captureId, { announce: false })" in script + assert script.index("selectCapture(captureId, { announce: false })") < script.index( + 'notice("success", "LABEL SAVED"' + ) + assert 'if (!activeCapture) els.reviewState.textContent = "READY";' in script + assert '"Capture is ARMED. Stop Session requires explicit confirmation."' in script + assert '"Capture is DISARMED. Start Session requires explicit confirmation."' in script + assert 'action === "start" ? els.stop' in script + assert 'action === "stop" ? els.start' in script + assert 'action === "delete" ? els.refresh' in script + assert "function focusIfAvailable(control)" in script + assert "focusIfAvailable(successFocus);" in script + assert "async function confirmed(action, fallbackFocus)" in script + assert "fallbackFocus?.focus();" in script + assert "function clearActiveDetail()" in script + assert 'if (action === "delete") clearActiveDetail();' in script + assert "captures.some((capture) => capture.capture_id === captureId)" in script + assert "refreshCaptures({ fallbackFocus: els.refresh })" in script + assert "focusIfAvailable(els.save);" in script diff --git a/tools/game-data-capture/tests/integration/test_runtime_cli.py b/tools/game-data-capture/tests/integration/test_runtime_cli.py new file mode 100644 index 0000000..e85579a --- /dev/null +++ b/tools/game-data-capture/tests/integration/test_runtime_cli.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import io +import subprocess +from collections import deque +from collections.abc import Iterable, Mapping +from contextlib import AbstractContextManager, nullcontext +from pathlib import Path +from threading import Event + +import pytest +import uvicorn + +import game_data_capture.cli as cli +import game_data_capture.runtime as runtime_module +from game_data_capture.config import AppConfig, LifecycleConfig, ServerConfig +from game_data_capture.ingest.clipboard import ( + ClipboardIntake, + FocusedWindowMetadata, + IntakeReceipt, + WatchConfig, +) +from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, receipt_name, spool_png +from game_data_capture.paths import RuntimePaths +from game_data_capture.runtime import CaptureRuntime, InstanceLock, RuntimeFailure, _CaptureControl +from game_data_capture.storage import CaptureRepository, ContentAddressedBlobStore, StorageDatabase + +_PNG = PNG_SIGNATURE + b"watch-png" + + +class FakeIntake: + def __init__(self, cancelled: Event, payloads: list[dict[str, object]]) -> None: + self.cancelled = cancelled + self.payloads = deque(payloads) + self.receipts: deque[IntakeReceipt] = deque() + self.armed = self.reconciled = self.shutdown_called = False + self.opened = 0 + self.acknowledged: list[str] = [] + self.seen_tokens: set[str] = set() + self._ticks = 0 + + def reconcile(self) -> list[str]: + self.reconciled = True + return [] + + def arm(self) -> None: + self.armed = True + + def tick(self, now: float | None = None) -> None: + self._ticks += 1 + if self._ticks > 1: + self.cancelled.set() + + def take_helper_receipt(self) -> dict[str, object] | None: + return self.payloads.popleft() if self.payloads else None + + def accept_helper_receipt( + self, + payload: Mapping[str, object], + *, + occurrence_id: str, + processing_version: str, + ) -> bool: + token = str(payload["receipt_token"]) + if token in self.seen_tokens: + return False + self.seen_tokens.add(token) + self.receipts.append( + IntakeReceipt( + occurrence_id, + token, + "0" * 64, + len(_PNG), + "image/png", + Path("/private/spool.png"), + "capture-v1", + processing_version, + FocusedWindowMetadata(None, None, None), + ) + ) + return True + + def take(self) -> IntakeReceipt | None: + return self.receipts.popleft() if self.receipts else None + + def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: + self.opened += 1 + return nullcontext([_PNG]) + + def acknowledge(self, receipt: IntakeReceipt) -> bool: + self.acknowledged.append(receipt.receipt_token) + return True + + def shutdown(self) -> None: + self.shutdown_called = True + + +def _runtime(tmp_path: Path, intake: FakeIntake) -> CaptureRuntime: + root = tmp_path / "xdg" + paths = RuntimePaths( + config_dir=(root / "config").resolve(), + data_dir=(root / "data").resolve(), + cache_dir=(root / "cache").resolve(), + runtime_dir=(root / "runtime").resolve(), + ) + return CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + paths, + migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + + +def _payload(token: str) -> dict[str, object]: + return { + "receipt_version": 1, + "receipt_token": token, + "sha256": "0" * 64, + "byte_size": len(_PNG), + "media_type": "image/png", + "spool_name": "ignored.png", + "policy_version": "capture-v1", + } + + +class PassiveClipboardIntake(ClipboardIntake): + """Use real private spool sidecars without launching a Wayland watcher.""" + + def __init__( + self, config: WatchConfig, cancelled: Event, *, acknowledge_result: bool = True + ) -> None: + super().__init__(config) + self.cancelled = cancelled + self.acknowledge_result = acknowledge_result + self.acknowledged: list[str] = [] + self.reconcile_calls = 0 + self._ticks = 0 + + def reconcile(self) -> list[str]: + self.reconcile_calls += 1 + return super().reconcile() + + def arm(self) -> None: + self.armed, self.paused = True, False + + def tick(self, now: float | None = None) -> None: + self._ticks += 1 + if self._ticks > 1: + self.cancelled.set() + + def acknowledge(self, receipt: IntakeReceipt) -> bool: + self.acknowledged.append(receipt.receipt_token) + return self.acknowledge_result and super().acknowledge(receipt) + + +class StopTimeoutReceiptIntake(PassiveClipboardIntake): + def __init__(self, config: WatchConfig, cancelled: Event) -> None: + super().__init__(config, cancelled) + self.tick_entered = Event() + self.release_tick = Event() + + def tick(self, now: float | None = None) -> None: + self.tick_entered.set() + self.release_tick.wait() + + def shutdown(self) -> None: + self.release_tick.set() + super().shutdown() + + +def _durable_runtime( + tmp_path: Path, cancelled: Event, *, acknowledge_result: bool = True +) -> tuple[CaptureRuntime, list[PassiveClipboardIntake]]: + created: list[PassiveClipboardIntake] = [] + root = tmp_path / "durable-xdg" + paths = RuntimePaths( + config_dir=(root / "config").resolve(), + data_dir=(root / "data").resolve(), + cache_dir=(root / "cache").resolve(), + runtime_dir=(root / "runtime").resolve(), + ) + + def factory(config: WatchConfig) -> PassiveClipboardIntake: + intake = PassiveClipboardIntake(config, cancelled, acknowledge_result=acknowledge_result) + created.append(intake) + return intake + + return ( + CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + paths, + migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(), + intake_factory=factory, + ), + created, + ) + + +def _durable_receipt(runtime: CaptureRuntime, token: str) -> None: + spool_png( + io.BytesIO(_PNG), + spool=runtime.paths.cache_dir / "spool", + max_bytes=100, + receipt_token=token, + policy_version="capture-v1", + ) + + +def test_watch_streams_verified_receipts_and_replays_exact_tokens_idempotently( + tmp_path: Path, +) -> None: + cancelled = Event() + intake = FakeIntake(cancelled, [_payload("watch-receipt-1"), _payload("watch-receipt-1")]) + runtime = _runtime(tmp_path, intake) + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir) + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + assert intake.armed and intake.reconciled and intake.shutdown_called and intake.opened == 1 + assert intake.acknowledged == ["watch-receipt-1"] + + +def test_watch_ignores_an_absent_helper_receipt_without_error_noise(tmp_path: Path) -> None: + cancelled = Event() + intake = FakeIntake(cancelled, []) + runtime = _runtime(tmp_path, intake) + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir) + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 0 + assert intake.shutdown_called and intake.opened == 0 + + +def test_startup_reconciles_partial_blob_storage_before_arming(tmp_path: Path) -> None: + cancelled = Event() + intake = FakeIntake(cancelled, []) + runtime = _runtime(tmp_path, intake) + blobs = ContentAddressedBlobStore(runtime.paths.data_dir) + blobs.temporary.mkdir(parents=True) + partial = blobs.temporary / ".leftover.partial" + partial.write_bytes(b"partial") + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + assert not partial.exists() + assert intake.reconciled and intake.shutdown_called + + +def test_durable_receipt_is_acknowledged_once_after_repository_confirmation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cancelled = Event() + runtime, created = _durable_runtime(tmp_path, cancelled) + _durable_receipt(runtime, "confirmed") + events: list[str] = [] + original_receive = CaptureRepository.receive + original_acknowledge = PassiveClipboardIntake.acknowledge + + def tracked_receive( + self: CaptureRepository, + capture_id: str, + receipt_token: str, + chunks: Iterable[bytes], + *, + media_type: str, + max_bytes: int, + ) -> str: + events.append("receive") + return original_receive( + self, + capture_id, + receipt_token, + chunks, + media_type=media_type, + max_bytes=max_bytes, + ) + + monkeypatch.setattr(CaptureRepository, "receive", tracked_receive) + + def tracked_acknowledge(self: PassiveClipboardIntake, receipt: IntakeReceipt) -> bool: + events.append("acknowledge") + return original_acknowledge(self, receipt) + + monkeypatch.setattr(PassiveClipboardIntake, "acknowledge", tracked_acknowledge) + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + + assert events == ["receive", "acknowledge"] + assert created[0].acknowledged == ["confirmed"] + assert created[0].reconcile_calls == 1 + assert not (runtime.paths.cache_dir / "spool" / receipt_name("confirmed")).exists() + + +def test_storage_failure_leaves_durable_receipt_unacknowledged( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cancelled = Event() + runtime, created = _durable_runtime(tmp_path, cancelled) + _durable_receipt(runtime, "storage-failure") + + def failed_receive( + self: CaptureRepository, + capture_id: str, + receipt_token: str, + chunks: Iterable[bytes], + *, + media_type: str, + max_bytes: int, + ) -> str: + raise OSError("storage unavailable") + + monkeypatch.setattr(CaptureRepository, "receive", failed_receive) + with pytest.raises(RuntimeFailure, match="capture watch failed"): + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + + assert created[0].acknowledged == [] + assert (runtime.paths.cache_dir / "spool" / receipt_name("storage-failure")).exists() + + +def test_cancellation_after_storage_confirmation_does_not_acknowledge( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cancelled = Event() + runtime, created = _durable_runtime(tmp_path, cancelled) + _durable_receipt(runtime, "cancelled-after-storage") + original_receive = CaptureRepository.receive + + def cancel_after_receive( + self: CaptureRepository, + capture_id: str, + receipt_token: str, + chunks: Iterable[bytes], + *, + media_type: str, + max_bytes: int, + ) -> str: + result = original_receive( + self, + capture_id, + receipt_token, + chunks, + media_type=media_type, + max_bytes=max_bytes, + ) + cancelled.set() + return result + + monkeypatch.setattr(CaptureRepository, "receive", cancel_after_receive) + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + + assert created[0].acknowledged == [] + assert (runtime.paths.cache_dir / "spool" / receipt_name("cancelled-after-storage")).exists() + + +def test_acknowledgement_failure_replays_idempotently_and_releases_lock(tmp_path: Path) -> None: + cancelled = Event() + runtime, failed = _durable_runtime(tmp_path, cancelled, acknowledge_result=False) + _durable_receipt(runtime, "replay-token") + with pytest.raises(RuntimeFailure, match="acknowledgement failed"): + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + assert failed[0].acknowledged == ["replay-token"] + sidecar = runtime.paths.cache_dir / "spool" / receipt_name("replay-token") + assert sidecar.exists() + reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + reusable.release() + + replay_cancelled = Event() + replay_runtime, replayed = _durable_runtime(tmp_path, replay_cancelled) + replay_runtime.watch(cancelled=replay_cancelled, poll_seconds=0.001) + database = StorageDatabase(replay_runtime.paths.data_dir, replay_runtime.migrations_dir) + with database.connect() as connection: + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 1 + assert replayed[0].acknowledged == ["replay-token"] + assert not sidecar.exists() + + +def test_service_stop_timeout_preserves_unacknowledged_durable_sidecar(tmp_path: Path) -> None: + cancelled = Event() + created: list[StopTimeoutReceiptIntake] = [] + root = tmp_path / "stop-timeout-xdg" + paths = RuntimePaths( + config_dir=(root / "config").resolve(), + data_dir=(root / "data").resolve(), + cache_dir=(root / "cache").resolve(), + runtime_dir=(root / "runtime").resolve(), + ) + + def factory(config: WatchConfig) -> StopTimeoutReceiptIntake: + intake = StopTimeoutReceiptIntake(config, cancelled) + created.append(intake) + return intake + + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0, shutdown_timeout_seconds=1)), + paths, + migrations_dir=(Path(__file__).parents[2] / "migrations").resolve(), + intake_factory=factory, + ) + _durable_receipt(runtime, "timeout-receipt") + resources = runtime._start() + control = _CaptureControl(runtime, resources.repository) + try: + assert control.start_session() == "started" + assert created[0].tick_entered.wait(0.2) + assert control.stop_session() == "stopped" + assert created[0].acknowledged == [] + assert (runtime.paths.cache_dir / "spool" / receipt_name("timeout-receipt")).exists() + finally: + control.shutdown() + resources.close() + + +def test_cli_bare_watch_is_a_safe_refusal_and_modes_never_start_live_processes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "fresh-xdg" + for key, name in ( + ("HOME", "home"), + ("XDG_CONFIG_HOME", "config"), + ("XDG_DATA_HOME", "data"), + ("XDG_CACHE_HOME", "cache"), + ("XDG_RUNTIME_DIR", "runtime"), + ): + monkeypatch.setenv(key, str(root / name)) + + def forbidden(*_arguments: object, **_kwargs: object) -> None: + raise AssertionError("test started a process") + + monkeypatch.setattr(subprocess, "Popen", forbidden) + assert cli.main(["watch"]) == cli.EXIT_UNIMPLEMENTED + assert "requires gdc-runtime-integration" in capsys.readouterr().err + assert cli.main(["watch", "--dry-run"]) == 0 + assert capsys.readouterr().out == "watch dry-run: capture runtime plan is valid\n" + assert not root.exists() + + +def test_cli_arm_reaches_only_a_mocked_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[str] = [] + + class FakeRuntime: + def __init__(self, _config: object, _paths: object) -> None: + calls.append("created") + + def watch(self) -> None: + calls.append("watch") + + monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime) + assert cli.main(["watch", "--arm"]) == 0 + assert calls == ["created", "watch"] + + +def test_cli_serve_validates_before_constructing_resources_and_passes_exact_authority( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, object]] = [] + + class FakeRuntime: + def __init__(self, config: AppConfig, _paths: object) -> None: + calls.append(("created", config.server)) + + def serve(self) -> None: + calls.append(("serve", None)) + + monkeypatch.setattr(cli, "CaptureRuntime", FakeRuntime) + assert cli.main(["serve", "--host", "127.0.0.1", "--port", "9877"]) == 0 + assert calls == [("created", ServerConfig("127.0.0.1", 9877)), ("serve", None)] + + calls.clear() + assert cli.main(["serve", "--host", "0.0.0.0"]) == cli.EXIT_CONFIG_ERROR + assert calls == [] + + +def test_cli_serve_converts_uvicorn_startup_exit_without_leaking_its_token( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "serve-xdg" + for key, name in ( + ("HOME", "home"), + ("XDG_CONFIG_HOME", "config"), + ("XDG_DATA_HOME", "data"), + ("XDG_CACHE_HOME", "cache"), + ("XDG_RUNTIME_DIR", "runtime"), + ): + monkeypatch.setenv(key, str(root / name)) + token = "Z" * 43 + monkeypatch.setattr(runtime_module, "new_bootstrap_token", lambda: token) + + def failed_server(*_: object, **__: object) -> None: + raise SystemExit(2) + + monkeypatch.setattr(uvicorn, "run", failed_server) + assert cli.main(["serve"]) == cli.EXIT_RUNTIME_ERROR + output = capsys.readouterr() + assert output.err == "capture runtime failed\n" + assert token not in output.err + assert "SystemExit" not in output.err + reusable = InstanceLock.acquire(root / "runtime" / "shd-capture" / "instance.lock", 0) + reusable.release() diff --git a/tools/game-data-capture/tests/integration/test_web_security.py b/tools/game-data-capture/tests/integration/test_web_security.py new file mode 100644 index 0000000..2173cf4 --- /dev/null +++ b/tools/game-data-capture/tests/integration/test_web_security.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from fastapi.testclient import TestClient + +from game_data_capture.config import ServerConfig +from game_data_capture.web.app import ( + CaptureRecord, + CaptureSessionStatus, + ControlOutcome, + MediaDescriptor, + MutationOutcome, + WebConfig, + create_app, +) +from game_data_capture.web.security import new_bootstrap_token + + +class Control: + def status(self) -> CaptureSessionStatus: + return CaptureSessionStatus(False, 0, 1) + + def start_session(self) -> ControlOutcome: + return "started" + + def stop_session(self) -> ControlOutcome: + return "stopped" + + +class EmptyCorpus: + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + return () + + def get_capture(self, capture_id: str) -> CaptureRecord | None: + return None + + def open_media(self, capture_id: str) -> MediaDescriptor | None: + return None + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: + return "not_found" + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: + return "not_found" + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: + return "not_found" + + +def test_bootstrap_is_one_time_same_origin_and_does_not_enable_permissive_cors() -> None: + token = new_bootstrap_token() + app = create_app( + config=WebConfig(ServerConfig()), + capture_control=Control(), + corpus=EmptyCorpus(), + bootstrap_token=token, + ) + client = TestClient(app, base_url="http://127.0.0.1:8765") + + cross_origin = client.post( + "/api/bootstrap", headers={"Origin": "http://evil.test"}, json={"token": token} + ) + accepted = client.post( + "/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token} + ) + replay = client.post( + "/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token} + ) + + assert cross_origin.status_code == 403 + assert accepted.status_code == 200 + assert replay.status_code == 401 + assert "access-control-allow-origin" not in accepted.headers diff --git a/tools/game-data-capture/tests/unit/ingest/test_clipboard.py b/tools/game-data-capture/tests/unit/ingest/test_clipboard.py new file mode 100644 index 0000000..6ba2e83 --- /dev/null +++ b/tools/game-data-capture/tests/unit/ingest/test_clipboard.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import io +import json +import math +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import game_data_capture.ingest.clipboard as clipboard +from game_data_capture.ingest.clipboard import ( + ClipboardIntake, + FocusedWindowMetadata, + WatchConfig, + focused_window, + restricted_environment, +) +from game_data_capture.ingest.spool_helper import PNG_SIGNATURE, spool_png + + +def receipt(spool: Path, token: str = "receipt") -> dict[str, object]: + return spool_png( + __import__("io").BytesIO(PNG_SIGNATURE + b"png"), + spool=spool, + max_bytes=100, + receipt_token=token, + policy_version="v1", + ) + + +def unique_receipt(spool: Path, token: str, value: bytes) -> dict[str, object]: + return spool_png( + io.BytesIO(PNG_SIGNATURE + value), + spool=spool, + max_bytes=100, + receipt_token=token, + policy_version="v1", + ) + + +def config(tmp_path: Path, *, queue: int = 2, retries: int = 2) -> WatchConfig: + return WatchConfig((tmp_path / "spool").resolve(), 100, queue, "v1", retries, 0.01, 0.1) + + +def test_environment_is_allowlisted_and_secret_free() -> None: + child = restricted_environment( + { + "PATH": "/bin", + "XDG_RUNTIME_DIR": "/run/user/1", + "OPENAI_API_KEY": "sk-secret", + "HTTP_AUTHORIZATION": "Bearer secret", + "PYTHONPATH": "/unsafe", + "UNRELATED": "no", + } + ) + assert child == {"PATH": "/bin", "XDG_RUNTIME_DIR": "/run/user/1"} + + +def test_text_clear_duplicate_and_backpressure_are_silent(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path, queue=1)) + item = receipt(intake.config.spool) + intake.armed = True + metadata = FocusedWindowMetadata(None, None, None) + assert not intake.accept_helper_receipt( + {"media_type": "text/plain"}, occurrence_id="a", processing_version="p" + ) + assert not intake.accept_helper_receipt({}, occurrence_id="a", processing_version="p") + assert intake.accept_helper_receipt( + item, occurrence_id="a", processing_version="p", metadata=metadata + ) + assert not intake.accept_helper_receipt( + item, occurrence_id="a", processing_version="p", metadata=metadata + ) + other = receipt(intake.config.spool, "other") + assert not intake.accept_helper_receipt( + other, occurrence_id="b", processing_version="p", metadata=metadata + ) + assert intake.queue_depth == 1 + + +def test_watcher_argv_places_options_before_watch_subcommand( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + launched: list[list[str]] = [] + + class Fake: + pid = os.getpid() + stdout = None + + def poll(self) -> int | None: + return None + + def wait(self, timeout: float | None = None) -> int: + return 0 + + def popen(args: list[str], **kwargs: object) -> Fake: + assert kwargs["shell"] is False + launched.append(args) + return Fake() + + monkeypatch.setattr(subprocess, "Popen", popen) + monkeypatch.setattr(os, "killpg", lambda _pid, _signal: None) + intake = ClipboardIntake(config(tmp_path)) + intake.arm() + command = launched[0] + watch = command.index("--watch") + assert command[:watch] == ["wl-paste", "--type", "image/png"] + assert command[watch + 1] == sys.executable + assert "--" not in command + assert "game_data_capture.ingest.spool_helper" in command + intake.pause() + assert intake.paused + intake.resume() + assert len(launched) == 2 + intake.disarm() + assert not intake.armed + + +def test_cancellation_disarms_and_clears_queue(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path)) + intake.armed = True + assert intake.accept_helper_receipt( + receipt(intake.config.spool), + occurrence_id="occurrence", + processing_version="processing-v1", + metadata=FocusedWindowMetadata(None, None, None), + ) + intake.cancel() + assert not intake.armed and intake.take() is None + + +def test_receipt_integrity_versions_and_bounded_duplicate_cache(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path, queue=2)) + intake.armed = True + metadata = FocusedWindowMetadata(None, None, None) + first = receipt(intake.config.spool, "one") + future = dict(first, receipt_version=2) + assert not intake.accept_helper_receipt( + future, occurrence_id="one", processing_version="p", metadata=metadata + ) + assert intake.accept_helper_receipt( + first, occurrence_id="one", processing_version="p", metadata=metadata + ) + assert not intake.accept_helper_receipt( + first, occurrence_id="one", processing_version="p", metadata=metadata + ) + assert len(intake._seen_tokens) == 1 + intake.shutdown() + assert not intake._seen_tokens and not intake._helper_payloads + + +def test_durable_receipt_recovery_is_null_metadata_duplicate_safe_and_acknowledged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + intake = ClipboardIntake(config(tmp_path)) + item = receipt(intake.config.spool, "crash-token") + intake.armed = True + monkeypatch.setattr( + clipboard, + "focused_window", + lambda: pytest.fail("recovered receipt must not query the current window"), + ) + intake.reconcile() + recovered = intake.take_helper_receipt() + assert recovered == item + assert intake.accept_helper_receipt( + recovered, occurrence_id="recovered", processing_version="p" + ) + accepted = intake.take() + assert accepted is not None + assert accepted.focused_window == FocusedWindowMetadata(None, None, None) + assert not intake.accept_helper_receipt(item, occurrence_id="live-copy", processing_version="p") + assert (intake.config.spool / "crash-token.receipt.json").exists() + assert intake.acknowledge(accepted) + assert not intake.acknowledge(accepted) + assert not (intake.config.spool / "crash-token.receipt.json").exists() + assert accepted.spool_path.exists() + + +def test_reconcile_respects_helper_backpressure_and_retains_durable_receipts( + tmp_path: Path, +) -> None: + intake = ClipboardIntake(config(tmp_path, queue=1)) + first = unique_receipt(intake.config.spool, "first", b"first") + second = unique_receipt(intake.config.spool, "second", b"second") + intake.armed = True + intake.reconcile() + queued = intake.take_helper_receipt() + assert queued in (first, second) + assert intake.accept_helper_receipt( + queued, + occurrence_id="first", + processing_version="p", + metadata=FocusedWindowMetadata(None, None, None), + ) + remaining = second if queued == first else first + assert (intake.config.spool / f"{remaining['receipt_token']}.receipt.json").exists() + intake.reconcile() + assert intake.take_helper_receipt() == remaining + + +def test_unacknowledged_receipt_survives_a_fresh_intake(tmp_path: Path) -> None: + initial = ClipboardIntake(config(tmp_path)) + item = receipt(initial.config.spool, "retry-token") + restarted = ClipboardIntake(config(tmp_path)) + restarted.armed = True + restarted.reconcile() + replay = restarted.take_helper_receipt() + assert replay == item + assert restarted.accept_helper_receipt(replay, occurrence_id="retry", processing_version="p") + accepted = restarted.take() + assert accepted is not None + assert (restarted.config.spool / "retry-token.receipt.json").exists() + + +@pytest.mark.parametrize("field", ["max_image_bytes", "max_queue_items", "max_restarts"]) +def test_bool_limits_are_rejected(tmp_path: Path, field: str) -> None: + values: dict[str, object] = dict( + spool=(tmp_path / "spool").resolve(), + max_image_bytes=100, + max_queue_items=1, + policy_version="v1", + max_restarts=1, + ) + values[field] = True + with pytest.raises(ValueError): + WatchConfig(**values) # type: ignore[arg-type] + + +def test_watcher_crash_uses_backoff_and_restart_cap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + starts = 0 + + class Dead: + pid = os.getpid() + stdout = None + + def poll(self) -> int: + return 1 + + def wait(self, timeout: float | None = None) -> int: + return 1 + + def popen(*_args: object, **_kwargs: object) -> Dead: + nonlocal starts + starts += 1 + return Dead() + + monkeypatch.setattr(subprocess, "Popen", popen) + intake = ClipboardIntake(config(tmp_path, retries=1)) + intake.arm() + intake.tick(0) + assert starts == 1 + intake.tick(0.02) + assert starts == 2 + intake.tick(0.02) + intake.tick(1) + assert starts == 2 + + +def test_niri_timeout_and_malformed_json_are_unknown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: None) + assert focused_window() == FocusedWindowMetadata(None, None, None) + monkeypatch.setattr(clipboard, "_bounded_command_output", lambda *_args: b"{broken") + assert focused_window() == FocusedWindowMetadata(None, None, None) + + +def test_receipt_mutation_matrix_and_file_integrity(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path)) + intake.armed = True + metadata = FocusedWindowMetadata(None, None, None) + base = unique_receipt(intake.config.spool, "token", b"base") + for key, value in ( + ("receipt_version", 2), + ("receipt_token", ""), + ("sha256", "0" * 64), + ("byte_size", 9), + ("media_type", "text/plain"), + ("policy_version", "other"), + ("spool_name", "wrong.png"), + ): + changed = dict(base) + changed[key] = value + assert not intake.accept_helper_receipt( + changed, occurrence_id="ok", processing_version="p", metadata=metadata + ) + for value in ("", "bad value"): + assert not intake.accept_helper_receipt( + base, occurrence_id=value, processing_version="p", metadata=metadata + ) + assert not intake.accept_helper_receipt( + base, occurrence_id="ok", processing_version=value, metadata=metadata + ) + for index, (mode, content) in enumerate( + ((0o600, b"changed"), (0o600, PNG_SIGNATURE), (0o644, None)) + ): + item = unique_receipt( + intake.config.spool, f"token{mode}{index}", bytes(f"{mode}{index}", "ascii") + ) + path = intake.config.spool / str(item["spool_name"]) + if content is not None: + path.write_bytes(content) + path.chmod(mode) + assert not intake.accept_helper_receipt( + item, occurrence_id=f"o{mode}", processing_version="p", metadata=metadata + ) + item = unique_receipt(intake.config.spool, "symlink", b"link") + path = intake.config.spool / str(item["spool_name"]) + path.unlink() + path.symlink_to(tmp_path / "outside") + assert not intake.accept_helper_receipt( + item, occurrence_id="symlink", processing_version="p", metadata=metadata + ) + + +def test_open_verified_and_acceptance_close_descriptors(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path)) + intake.armed = True + item = unique_receipt(intake.config.spool, "fd", b"reader") + before = len(list(Path("/proc/self/fd").iterdir())) + assert intake.accept_helper_receipt( + item, + occurrence_id="fd", + processing_version="p", + metadata=FocusedWindowMetadata(None, None, None), + ) + accepted = intake.take() + assert accepted is not None and len(list(Path("/proc/self/fd").iterdir())) == before + with intake.open_verified(accepted) as reader: + assert b"".join(reader) == PNG_SIGNATURE + b"reader" + path = accepted.spool_path + path.write_bytes(b"changed") + with pytest.raises(ValueError), intake.open_verified(accepted): + pass + + +def test_duplicate_cache_eviction_shutdown_and_unreferenced_candidates(tmp_path: Path) -> None: + intake = ClipboardIntake(config(tmp_path, queue=2)) + intake.armed = True + metadata = FocusedWindowMetadata(None, None, None) + items = [unique_receipt(intake.config.spool, f"t{i}", bytes([i])) for i in range(3)] + for index, item in enumerate(items): + assert intake.accept_helper_receipt( + item, occurrence_id=f"o{index}", processing_version="p", metadata=metadata + ) + intake.take() + assert len(intake._seen_tokens) == 2 and not intake.accept_helper_receipt( + items[2], occurrence_id="latest", processing_version="p", metadata=metadata + ) + assert intake.accept_helper_receipt( + items[0], occurrence_id="evicted", processing_version="p", metadata=metadata + ) + intake = ClipboardIntake(config(tmp_path / "other", queue=1)) + intake.armed = True + queued = unique_receipt(intake.config.spool, "queued", b"queued") + extra = unique_receipt(intake.config.spool, "extra", b"extra") + assert intake.accept_helper_receipt( + queued, occurrence_id="queued", processing_version="p", metadata=metadata + ) + assert [path.name for path in intake.unreferenced_spool_candidates()] == [ + str(extra["spool_name"]) + ] + intake._helper_payloads.append({"receipt": "real"}) + intake.shutdown() + assert ( + not intake._queue + and not intake._helper_payloads + and not intake._seen_tokens + and not intake._seen_order + ) + + +def test_receipt_reader_bounds_spawn_failures_and_config_types( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class Output: + def __init__(self) -> None: + self.parts = [ + b"x" * 1025 + b"\n", + json.dumps({"ok": 1}).encode() + b"\n", + json.dumps({"two": 2}).encode() + b"\n", + ] + + def readline(self, _size: int) -> bytes: + return self.parts.pop(0) if self.parts else b"" + + class Process: + stdout = Output() + + intake = ClipboardIntake(config(tmp_path, queue=1)) + intake._read_receipts(Process()) # type: ignore[arg-type] + assert intake.helper_queue_depth == 1 and intake.take_helper_receipt() == {"ok": 1} + attempts = 0 + + def failing(*_args: object, **_kwargs: object) -> object: + nonlocal attempts + attempts += 1 + raise OSError("missing") + + monkeypatch.setattr(subprocess, "Popen", failing) + failed = ClipboardIntake(config(tmp_path, retries=2)) + failed.arm() + failed.tick(failed._restart_after + 0.01) + failed.tick(failed._restart_after + 0.01) + failed.tick(failed._restart_after + 1) + assert attempts == 3 + for field, value in ( + ("max_image_bytes", 1.5), + ("max_queue_items", "1"), + ("max_restarts", 1.5), + ("shutdown_timeout_seconds", 0), + ("restart_backoff_seconds", math.inf), + ): + values: dict[str, object] = dict( + spool=(tmp_path / "x").resolve(), + max_image_bytes=100, + max_queue_items=1, + policy_version="v1", + max_restarts=1, + ) + values[field] = value + with pytest.raises(ValueError): + WatchConfig(**values) # type: ignore[arg-type] + + +def test_bounded_niri_runner_and_valid_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + assert ( + clipboard._bounded_command_output([sys.executable, "-c", "print('x'*20000)"], {}, 1) is None + ) + assert ( + clipboard._bounded_command_output( + [sys.executable, "-c", "import time; time.sleep(2)"], {}, 0.01 + ) + is None + ) + monkeypatch.setattr( + clipboard, + "_bounded_command_output", + lambda *_args: b'{"app_id":"game","title":"title","id":1}', + ) + assert focused_window() == FocusedWindowMetadata("game", "title", 1) + + +def test_exited_watcher_is_waited_stdout_closed_and_reader_retired(tmp_path: Path) -> None: + class Output: + closed = False + + def close(self) -> None: + self.closed = True + + class Child: + pid = os.getpid() + stdout = Output() + waits = 0 + + def poll(self) -> int: + return 1 + + def wait(self, timeout: float | None = None) -> int: + self.waits += 1 + return 1 + + class Reader: + joined = False + + def join(self, timeout: float | None = None) -> None: + self.joined = True + + intake = ClipboardIntake(config(tmp_path)) + child = Child() + reader = Reader() + intake._watcher = child # type: ignore[assignment] + intake._reader = reader # type: ignore[assignment] + intake.pause() + assert child.waits == 1 and child.stdout.closed and reader.joined and intake._reader is None diff --git a/tools/game-data-capture/tests/unit/ingest/test_spool_helper.py b/tools/game-data-capture/tests/unit/ingest/test_spool_helper.py new file mode 100644 index 0000000..76c1595 --- /dev/null +++ b/tools/game-data-capture/tests/unit/ingest/test_spool_helper.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import io +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import game_data_capture.ingest.spool_helper as spool_helper +from game_data_capture.ingest.spool_helper import ( + PNG_SIGNATURE, + SpoolError, + acknowledge_receipt, + durable_receipts, + receipt_name, + reconcile_spool, + spool_png, +) + + +def png(payload: bytes = b"data") -> bytes: + return PNG_SIGNATURE + payload + + +def fill_receipt_capacity(spool: Path) -> dict[str, object]: + seed = spool_png( + io.BytesIO(png(b"seed")), + spool=spool, + max_bytes=100, + receipt_token="seed", + policy_version="v1", + ) + for index in range(255): + token = f"pre{index}" + sidecar = spool / receipt_name(token) + sidecar.write_text( + json.dumps(dict(seed, receipt_token=token), sort_keys=True, separators=(",", ":")) + ) + sidecar.chmod(0o600) + return seed + + +def test_png_spool_is_private_atomic_and_deduplicated( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + spool = (tmp_path / "spool").resolve() + replaced: list[tuple[Path, Path]] = [] + real_replace = os.replace + + def replace(source: Path, destination: Path) -> None: + replaced.append((source, destination)) + real_replace(source, destination) + + monkeypatch.setattr(os, "replace", replace) + first = spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1" + ) + second = spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="two", policy_version="v1" + ) + assert first["sha256"] == second["sha256"] + assert (spool.stat().st_mode & 0o777) == 0o700 + installed = spool / str(first["spool_name"]) + assert installed.read_bytes() == png() and (installed.stat().st_mode & 0o777) == 0o600 + assert len(replaced) == 1 and replaced[0][1] == installed + sidecars = sorted(spool.glob("*.receipt.json")) + assert [sidecar.name for sidecar in sidecars] == ["one.receipt.json", "two.receipt.json"] + assert all((sidecar.stat().st_mode & 0o777) == 0o600 for sidecar in sidecars) + assert all(set(json.loads(sidecar.read_text())) == set(first) for sidecar in sidecars) + assert not list(spool.glob("*.partial")) + + +@pytest.mark.parametrize("value", [b"", b"text", PNG_SIGNATURE[:4]]) +def test_empty_text_and_malformed_png_leave_no_partial(tmp_path: Path, value: bytes) -> None: + spool = (tmp_path / "spool").resolve() + with pytest.raises(SpoolError): + spool_png( + io.BytesIO(value), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1" + ) + assert not list(spool.glob("*.partial")) + + +def test_unsupported_mime_leaves_no_partial(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + with pytest.raises(SpoolError): + spool_png( + io.BytesIO(png()), + spool=spool, + max_bytes=100, + receipt_token="one", + policy_version="v1", + media_type="text/plain", + ) + assert not spool.exists() + + +def test_short_reads_accumulate_the_png_signature(tmp_path: Path) -> None: + class ShortReads: + def __init__(self) -> None: + self.parts = [PNG_SIGNATURE[:3], PNG_SIGNATURE[3:] + b"payload", b""] + + def read(self, _size: int) -> bytes: + return self.parts.pop(0) + + receipt = spool_png( + ShortReads(), + spool=(tmp_path / "spool").resolve(), + max_bytes=100, + receipt_token="one", + policy_version="v1", + ) + assert receipt["byte_size"] == len(PNG_SIGNATURE + b"payload") + + +def test_size_bound_and_crash_reconciliation_remove_partials(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + with pytest.raises(SpoolError): + spool_png( + io.BytesIO(png(b"x" * 100)), + spool=spool, + max_bytes=10, + receipt_token="one", + policy_version="v1", + ) + leftover = spool / ".crash.partial" + leftover.write_bytes(b"incomplete") + assert reconcile_spool(spool) == [".crash.partial"] + assert not leftover.exists() + + +def test_pending_receipt_is_promoted_only_when_its_png_is_verified(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + receipt = spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1" + ) + final = spool / receipt_name("one") + pending = spool / ".one.receipt.pending" + os.replace(final, pending) + assert reconcile_spool(spool) == [] + assert final.exists() and not pending.exists() + final.unlink() + pending.write_text(json.dumps(receipt)) + assert reconcile_spool(spool) == [pending.name] + assert not pending.exists() + + +def test_receipt_token_replay_cannot_replace_the_original_sidecar(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + original = spool_png( + io.BytesIO(png(b"original")), + spool=spool, + max_bytes=100, + receipt_token="same-token", + policy_version="v1", + ) + final = spool / receipt_name("same-token") + original_bytes = final.read_bytes() + assert ( + spool_png( + io.BytesIO(png(b"original")), + spool=spool, + max_bytes=100, + receipt_token="same-token", + policy_version="v1", + ) + == original + ) + with pytest.raises(SpoolError, match="receipt token conflict"): + spool_png( + io.BytesIO(png(b"different")), + spool=spool, + max_bytes=100, + receipt_token="same-token", + policy_version="v1", + ) + assert final.read_bytes() == original_bytes + conflicting = dict(original, sha256="0" * 64, spool_name=f"{'0' * 64}.png") + pending = spool / ".same-token.receipt.pending" + pending.write_text(json.dumps(conflicting, sort_keys=True, separators=(",", ":"))) + pending.chmod(0o600) + assert pending.name in reconcile_spool(spool) + assert final.read_bytes() == original_bytes and not pending.exists() + + +def test_sidecars_are_bounded_validated_and_acknowledged_exactly_once(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + first = spool_png( + io.BytesIO(png(b"one")), + spool=spool, + max_bytes=100, + receipt_token="one", + policy_version="v1", + ) + second = spool_png( + io.BytesIO(png(b"two")), + spool=spool, + max_bytes=100, + receipt_token="two", + policy_version="v1", + ) + malformed = spool / "bad.receipt.json" + malformed.write_bytes(b"{" + b"x" * 1024) + malformed.chmod(0o600) + linked = spool / "linked.receipt.json" + linked.symlink_to(malformed) + assert {item["receipt_token"] for item in durable_receipts(spool, max_items=2)} == { + "one", + "two", + } + assert acknowledge_receipt(spool, first) + assert not acknowledge_receipt(spool, first) + assert (spool / receipt_name("two")).exists() + assert not acknowledge_receipt(spool, dict(second, receipt_token="one")) + + +def test_malformed_sidecars_do_not_starve_valid_receipts_or_partial_cleanup(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + spool.mkdir(mode=0o700) + for index in range(256): + malformed = spool / f"{index:03d}.receipt.json" + malformed.write_bytes(b"{") + malformed.chmod(0o600) + valid = spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="valid", policy_version="v1" + ) + assert list(durable_receipts(spool, max_items=1)) == [valid] + assert list(spool.glob("*.receipt.json")) == [spool / "valid.receipt.json"] + for index in range(256): + pending = spool / f".pending{index}.receipt.pending" + pending.write_text( + json.dumps( + dict(valid, receipt_token=f"pending{index}"), sort_keys=True, separators=(",", ":") + ) + ) + pending.chmod(0o600) + partial = spool / ".after-pending.partial" + partial.write_bytes(b"incomplete") + assert partial.name in reconcile_spool(spool) + assert not partial.exists() + + +def test_receipt_capacity_rejects_new_tokens_but_accepts_exact_replay(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + seed = fill_receipt_capacity(spool) + evidence = (spool / receipt_name("seed")).read_bytes() + png_names = {path.name for path in spool.glob("*.png")} + with pytest.raises(SpoolError, match="capacity"): + spool_png( + io.BytesIO(png(b"new")), + spool=spool, + max_bytes=100, + receipt_token="new", + policy_version="v1", + ) + assert (spool / receipt_name("seed")).read_bytes() == evidence + assert {path.name for path in spool.glob("*.png")} == png_names + assert ( + spool_png( + io.BytesIO(png(b"seed")), + spool=spool, + max_bytes=100, + receipt_token="seed", + policy_version="v1", + ) + == seed + ) + + +def test_receipt_capacity_reconciles_malformed_files_before_counting(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + spool.mkdir(mode=0o700) + for index in range(256): + malformed = spool / f"bad{index}.receipt.json" + malformed.write_bytes(b"{") + malformed.chmod(0o600) + accepted = spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="accepted", policy_version="v1" + ) + assert accepted["receipt_token"] == "accepted" + assert list(spool.glob("bad*.receipt.json")) == [] + + +def test_receipt_capacity_is_serialized_across_helper_processes(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + fill_receipt_capacity(spool) + for index in range(1): + (spool / receipt_name(f"pre{index}")).unlink() + arguments = [ + sys.executable, + "-m", + "game_data_capture.ingest.spool_helper", + "--spool", + str(spool), + "--max-bytes", + "100", + "--policy-version", + "v1", + ] + first = subprocess.Popen([*arguments, "--receipt-token", "race-one"], stdin=subprocess.PIPE) + second = subprocess.Popen([*arguments, "--receipt-token", "race-two"], stdin=subprocess.PIPE) + assert first.stdin is not None and second.stdin is not None + first.stdin.write(png(b"one")) + first.stdin.close() + second.stdin.write(png(b"two")) + second.stdin.close() + assert sorted((first.wait(), second.wait())) == [0, 2] + assert len(list(spool.glob("*.receipt.json"))) == 256 + + +def test_spool_lock_is_private_rejects_links_and_releases_on_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + spool = (tmp_path / "spool").resolve() + spool.mkdir(mode=0o700) + (spool / ".spool.lock").symlink_to(tmp_path / "outside") + with pytest.raises(SpoolError, match="unsafe spool lock"): + spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="link", policy_version="v1" + ) + (spool / ".spool.lock").unlink() + with monkeypatch.context() as context: + context.setattr( + spool_helper, + "_install_pending_sidecar", + lambda *_args: (_ for _ in ()).throw(OSError()), + ) + with pytest.raises(OSError): + spool_png( + io.BytesIO(png()), + spool=spool, + max_bytes=100, + receipt_token="failure", + policy_version="v1", + ) + accepted = spool_png( + io.BytesIO(png()), + spool=spool, + max_bytes=100, + receipt_token="recovered", + policy_version="v1", + ) + assert accepted["receipt_token"] == "recovered" + assert (spool / ".spool.lock").stat().st_mode & 0o777 == 0o600 + + +def test_short_write_removes_partial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + spool = (tmp_path / "spool").resolve() + monkeypatch.setattr(os, "write", lambda _fd, _data: 0) + with pytest.raises(OSError): + spool_png( + io.BytesIO(png()), spool=spool, max_bytes=100, receipt_token="one", policy_version="v1" + ) + assert not list(spool.glob("*.partial")) + + +def test_helper_source_has_no_service_or_storage_imports() -> None: + source = Path(__file__).parents[3] / "src/game_data_capture/ingest/spool_helper.py" + text = source.read_text(encoding="utf-8") + for forbidden in ("fastapi", "sqlite", "openai", "game_data_capture.storage"): + assert forbidden not in text.lower() + + +def test_stdin_entrypoint_returns_only_bounded_json_receipt(tmp_path: Path) -> None: + spool = (tmp_path / "spool").resolve() + completed = subprocess.run( + [ + sys.executable, + "-m", + "game_data_capture.ingest.spool_helper", + "--spool", + str(spool), + "--max-bytes", + "100", + "--receipt-token", + "receipt", + "--policy-version", + "v1", + ], + input=png(), + capture_output=True, + check=False, + env={"PATH": os.environ.get("PATH", "")}, + ) + assert completed.returncode == 0 and len(completed.stdout) <= 1024 and not completed.stderr + receipt = json.loads(completed.stdout) + assert receipt["media_type"] == "image/png" and receipt["receipt_token"] == "receipt" diff --git a/tools/game-data-capture/tests/unit/pipeline/test_runtime.py b/tools/game-data-capture/tests/unit/pipeline/test_runtime.py new file mode 100644 index 0000000..79dd5df --- /dev/null +++ b/tools/game-data-capture/tests/unit/pipeline/test_runtime.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable, Iterable, Mapping +from contextlib import AbstractContextManager +from pathlib import Path +from threading import Event, Thread +from typing import cast + +import pytest +import uvicorn +from fastapi.testclient import TestClient +from starlette.types import ASGIApp + +import game_data_capture.runtime as runtime_module +from game_data_capture.config import AppConfig, CaptureLimits, LifecycleConfig, ServerConfig +from game_data_capture.ingest.clipboard import IntakeReceipt +from game_data_capture.ingest.spool_helper import PNG_SIGNATURE +from game_data_capture.paths import RuntimePaths +from game_data_capture.runtime import ( + CaptureRuntime, + InstanceLock, + IntakePort, + RuntimeFailure, + _CaptureControl, +) +from game_data_capture.storage import CaptureCorpus, CaptureRepository, StorageDatabase + +_PNG = PNG_SIGNATURE + b"unit-png" + + +def _paths(tmp_path: Path) -> RuntimePaths: + root = tmp_path / "xdg" + return RuntimePaths( + config_dir=(root / "config").resolve(), + data_dir=(root / "data").resolve(), + cache_dir=(root / "cache").resolve(), + runtime_dir=(root / "runtime").resolve(), + ) + + +def _runtime(tmp_path: Path, *, limits: CaptureLimits | None = None) -> CaptureRuntime: + return CaptureRuntime( + AppConfig( + lifecycle=LifecycleConfig(lock_timeout_seconds=0), limits=limits or CaptureLimits() + ), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + ) + + +def test_one_shot_persists_distinct_occurrences_and_one_blob(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + image = tmp_path / "image.png" + image.write_bytes(_PNG) + + first = runtime.ingest_image(image) + second = runtime.ingest_image(image) + database = StorageDatabase(runtime.paths.data_dir, runtime.migrations_dir) + + with database.connect() as connection: + assert first != second + assert connection.execute("SELECT count(*) FROM captures").fetchone()[0] == 2 + assert connection.execute("SELECT count(*) FROM blobs").fetchone()[0] == 1 + + +@pytest.mark.parametrize("kind", ["malformed", "oversize", "symlink", "broad-mode"]) +def test_one_shot_rejects_unsafe_png_inputs(tmp_path: Path, kind: str) -> None: + runtime = _runtime(tmp_path, limits=CaptureLimits(max_image_bytes=len(_PNG))) + image = tmp_path / "image.png" + image.write_bytes(_PNG) + if kind == "malformed": + image.write_bytes(b"not-a-png") + elif kind == "oversize": + image.write_bytes(_PNG + b"x") + elif kind == "symlink": + linked = tmp_path / "linked.png" + linked.symlink_to(image) + image = linked + else: + image.chmod(0o666) + try: + with pytest.raises(RuntimeFailure, match="safe PNG"): + runtime.ingest_image(image) + finally: + if image.exists() and not image.is_symlink(): + image.chmod(0o600) + + +def test_stale_lock_file_is_reused_but_an_active_lock_is_not(tmp_path: Path) -> None: + paths = _paths(tmp_path) + paths.runtime_dir.mkdir(parents=True) + paths.instance_lock_file.write_text("stale", encoding="utf-8") + paths.instance_lock_file.chmod(0o600) + first = InstanceLock.acquire(paths.instance_lock_file, 0) + try: + with pytest.raises(RuntimeFailure, match="another capture instance is active"): + InstanceLock.acquire(paths.instance_lock_file, 0) + finally: + first.release() + second = InstanceLock.acquire(paths.instance_lock_file, 0) + second.release() + assert paths.instance_lock_file.exists() + + +class InterruptingIntake: + def __init__(self) -> None: + self.shutdown_called = False + + def reconcile(self) -> list[str]: + return [] + + def arm(self) -> None: + return None + + def tick(self) -> None: + raise KeyboardInterrupt + + def take_helper_receipt(self) -> dict[str, object] | None: + return None + + def accept_helper_receipt( + self, payload: Mapping[str, object], *, occurrence_id: str, processing_version: str + ) -> bool: + return False + + def take(self) -> IntakeReceipt | None: + return None + + def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: + raise AssertionError("no receipt should be opened") + + def acknowledge(self, receipt: IntakeReceipt) -> bool: + raise AssertionError("no receipt should be acknowledged") + + def shutdown(self) -> None: + self.shutdown_called = True + + +def test_keyboard_interrupt_reaps_intake_and_releases_runtime_lock(tmp_path: Path) -> None: + intake = InterruptingIntake() + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + runtime.watch(poll_seconds=0.001) + + assert intake.shutdown_called + reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + reusable.release() + + +def test_cancelled_watch_cleans_up_without_entering_the_tick_loop(tmp_path: Path) -> None: + cancelled = Event() + cancelled.set() + intake = InterruptingIntake() + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + runtime.watch(cancelled=cancelled, poll_seconds=0.001) + assert intake.shutdown_called + + +def test_one_shot_never_connects_outside_the_utility_data_root( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + runtime = _runtime(tmp_path) + image = tmp_path / "image.png" + image.write_bytes(_PNG) + real_connect = cast(Callable[..., sqlite3.Connection], sqlite3.connect) + connected: list[Path] = [] + + def checked_connect(database: str | Path, **kwargs: object) -> sqlite3.Connection: + connected.append(Path(database)) + return real_connect(database, **kwargs) + + monkeypatch.setattr(sqlite3, "connect", checked_connect) + runtime.ingest_image(image) + assert connected + assert all(path.parent == runtime.paths.data_dir for path in connected) + + +class ControlledIntake: + def __init__(self, *, fail_tick: bool = False) -> None: + self.armed = False + self.reconciled = False + self.shutdown_called = Event() + self.queue_depth = 0 + self.helper_queue_depth = 0 + self.fail_tick = fail_tick + + def reconcile(self) -> list[str]: + self.reconciled = True + return [] + + def arm(self) -> None: + self.armed = True + + def tick(self) -> None: + if self.fail_tick: + raise OSError("helper fault") + + def take_helper_receipt(self) -> dict[str, object] | None: + return None + + def accept_helper_receipt( + self, payload: Mapping[str, object], *, occurrence_id: str, processing_version: str + ) -> bool: + return False + + def take(self) -> IntakeReceipt | None: + return None + + def open_verified(self, receipt: IntakeReceipt) -> AbstractContextManager[Iterable[bytes]]: + raise AssertionError("no receipt should be opened") + + def acknowledge(self, receipt: IntakeReceipt) -> bool: + raise AssertionError("no receipt should be acknowledged") + + def shutdown(self) -> None: + self.shutdown_called.set() + + +class FaultingCleanupIntake(ControlledIntake): + def __init__(self) -> None: + super().__init__(fail_tick=True) + self.cleanup_started = Event() + self.cleanup_release = Event() + self._shutdown_calls = 0 + + def shutdown(self) -> None: + self._shutdown_calls += 1 + self.cleanup_started.set() + if self._shutdown_calls == 1: + self.cleanup_release.wait() + self.shutdown_called.set() + + +class MalformedQueueIntake: + @property + def queue_depth(self) -> object: + raise OSError("private intake failure") + + @property + def helper_queue_depth(self) -> object: + return True + + +class EmptyRepository: + def receive( + self, + capture_id: str, + receipt_token: str, + chunks: Iterable[bytes], + *, + media_type: str, + max_bytes: int, + ) -> str: + raise AssertionError("no receipt should be stored") + + +def test_control_serializes_repeated_start_stop_and_reports_bounded_queue(tmp_path: Path) -> None: + intake = ControlledIntake() + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository())) + outcomes: list[str] = [] + + def start() -> None: + outcomes.append(control.start_session()) + + first, second = Thread(target=start), Thread(target=start) + first.start() + second.start() + first.join() + second.join() + + assert sorted(outcomes) == ["already_armed", "started"] + assert intake.reconciled and intake.armed + assert control.status().queue_depth == 0 + assert control.status().queue_capacity == runtime.config.limits.max_queue_items + assert control.stop_session() in {"stopped", "already_stopped"} + assert control.stop_session() == "already_stopped" + assert intake.shutdown_called.wait(0.1) + + +def test_control_status_combines_and_bounds_both_intake_queues(tmp_path: Path) -> None: + intake = ControlledIntake() + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository())) + + intake.queue_depth, intake.helper_queue_depth = 2, 3 + control._intake = cast(IntakePort, intake) + assert control.status().queue_depth == 5 + intake.queue_depth, intake.helper_queue_depth = 1000, -1 + assert control.status().queue_depth == runtime.config.limits.max_queue_items + control._intake = cast(IntakePort, MalformedQueueIntake()) + assert control.status().queue_depth == 0 + + +def test_control_turns_background_intake_fault_into_bounded_disarmed_state(tmp_path: Path) -> None: + intake = ControlledIntake(fail_tick=True) + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + control = _CaptureControl(runtime, cast(CaptureRepository, EmptyRepository())) + + assert control.start_session() == "started" + assert intake.shutdown_called.wait(0.2) + assert control.stop_session() in {"stopped", "already_stopped"} + assert control.status().armed is False + assert control.status().queue_depth == 0 + + +def test_service_retains_its_lock_until_faulting_worker_cleanup_finishes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + intake = FaultingCleanupIntake() + runtime = CaptureRuntime( + AppConfig(lifecycle=LifecycleConfig(lock_timeout_seconds=0, shutdown_timeout_seconds=1)), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=lambda _config: intake, + ) + captured: dict[str, _CaptureControl] = {} + + def factory(**kwargs: object) -> object: + captured["control"] = cast(_CaptureControl, kwargs["capture_control"]) + return object() + + def runner(_: object, **__: object) -> None: + assert captured["control"].start_session() == "started" + assert intake.cleanup_started.wait(0.2) + + monkeypatch.setattr(runtime_module, "create_app", factory) + monkeypatch.setattr(uvicorn, "run", runner) + failure: list[RuntimeFailure] = [] + + def serve() -> None: + try: + runtime.serve() + except RuntimeFailure as error: + failure.append(error) + + service = Thread(target=serve) + service.start() + try: + service.join(timeout=2) + assert not service.is_alive() + assert failure + with pytest.raises(RuntimeFailure, match="another capture instance"): + InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + outcomes: list[str] = [] + start = Thread(target=lambda: outcomes.append(captured["control"].start_session())) + stop = Thread(target=lambda: outcomes.append(captured["control"].stop_session())) + start.start() + stop.start() + start.join() + stop.join() + assert sorted(outcomes) == ["conflict", "conflict"] + finally: + intake.cleanup_release.set() + for _ in range(50): + try: + reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + except RuntimeFailure: + Event().wait(0.01) + else: + reusable.release() + break + else: + pytest.fail("service lock was not released after worker cleanup") + + +def test_serve_constructs_durable_corpus_without_auto_arming_and_uses_one_fragment_token( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + created: list[ControlledIntake] = [] + token = "A" * 43 + + def factory(_: object) -> ControlledIntake: + intake = ControlledIntake() + created.append(intake) + return intake + + runtime = CaptureRuntime( + AppConfig( + server=ServerConfig(host="127.0.0.1", port=9876), + lifecycle=LifecycleConfig(lock_timeout_seconds=0), + ), + _paths(tmp_path), + migrations_dir=(Path(__file__).parents[3] / "migrations").resolve(), + intake_factory=factory, + ) + monkeypatch.setattr(runtime_module, "new_bootstrap_token", lambda: token) + received: dict[str, object] = {} + + def runner(app: object, **kwargs: object) -> None: + received.update(kwargs) + with TestClient(cast(ASGIApp, app), base_url="http://127.0.0.1:9876") as client: + assert client.get("/health").json() == {"status": "ok"} + assert client.get("/").status_code == 200 + assert created == [] + bootstrapped = client.post( + "/api/bootstrap", + json={"token": token}, + headers={"Origin": "http://127.0.0.1:9876"}, + ) + csrf = bootstrapped.json()["csrf_token"] + assert client.get("/api/session").json()["armed"] is False + started = client.post( + "/api/session/start", + json={"confirmation": "START"}, + headers={"Origin": "http://127.0.0.1:9876", "X-CSRF-Token": csrf}, + ) + assert started.json()["result"] == "started" + assert created[0].reconciled and created[0].armed + + monkeypatch.setattr(uvicorn, "run", runner) + runtime.serve() + + output = capsys.readouterr().out + assert output.count(token) == 1 + assert f"http://127.0.0.1:9876/#bootstrap={token}" in output + assert received == { + "host": "127.0.0.1", + "port": 9876, + "reload": False, + "workers": 1, + "proxy_headers": False, + } + assert created[0].shutdown_called.wait(0.1) + reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + reusable.release() + + +def test_service_startup_reconciliation_failure_releases_the_instance_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + runtime = _runtime(tmp_path) + + def failed_reconciliation(self: CaptureCorpus, *, limit: int = 64) -> int: + raise OSError("private deletion recovery fault") + + monkeypatch.setattr(CaptureCorpus, "reconcile_deletions", failed_reconciliation) + with pytest.raises(RuntimeFailure, match="initialization failed"): + runtime.serve() + reusable = InstanceLock.acquire(runtime.paths.instance_lock_file, 0) + reusable.release() diff --git a/tools/game-data-capture/tests/unit/storage/test_blobs.py b/tools/game-data-capture/tests/unit/storage/test_blobs.py new file mode 100644 index 0000000..ffdc60e --- /dev/null +++ b/tools/game-data-capture/tests/unit/storage/test_blobs.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import hashlib +import os +import threading +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from game_data_capture.storage.blobs import ( + BlobCorruptionError, + BlobLimitError, + BlobLockError, + ContentAddressedBlobStore, +) + + +def test_blob_is_verified_deduplicated_and_private(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + payload = b"capture-bytes" + digest, size = store.put([payload], max_bytes=100) + assert digest == hashlib.sha256(payload).hexdigest() + assert store.put([payload], max_bytes=100) == (digest, size) + assert b"".join(store.read(digest, max_bytes=100)) == payload + assert (store._path(digest).stat().st_mode & 0o777) == 0o600 + + +def test_corruption_and_partial_files_are_detected_or_reconciled(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + digest, _ = store.put([b"good"], max_bytes=100) + store._path(digest).write_bytes(b"bad") + with pytest.raises(BlobCorruptionError): + store.verify(digest) + store.temporary.mkdir(parents=True, exist_ok=True) + partial = store.temporary / ".crash.partial" + partial.write_bytes(b"partial") + assert store.reconcile() == [".crash.partial"] + assert not partial.exists() + + +def test_rejects_hash_and_generator_failures_without_partial_file(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + with pytest.raises(BlobCorruptionError): + store.put([b"value"], max_bytes=100, expected_sha256="0" * 64) + + def broken() -> Iterator[bytes]: + yield b"prefix" + raise RuntimeError("source failed") + + with pytest.raises(RuntimeError): + store.put(broken(), max_bytes=100) + assert not list(store.temporary.glob("*.partial")) + + +def test_root_symlinks_are_not_traversed(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + root = (tmp_path / "data").resolve() + root.symlink_to(outside, target_is_directory=True) + store = ContentAddressedBlobStore(root) + with pytest.raises(BlobCorruptionError): + store.put([b"x"], max_bytes=1) + + +def test_limits_and_root_symlink_reconciliation_are_safe(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + for limit in (0, -1, True): + with pytest.raises(BlobLimitError): + store.put([b"x"], max_bytes=limit) + digest, _ = store.put([b"x"], max_bytes=1) + for limit in (0, -1, True): + with pytest.raises(BlobLimitError): + list(store.read(digest, max_bytes=limit)) + outside = tmp_path / "outside" + outside.mkdir() + partial = outside / ".outside.partial" + partial.write_bytes(b"outside") + store.temporary.rmdir() + store.temporary.symlink_to(outside, target_is_directory=True) + assert store.reconcile() == [] + assert partial.read_bytes() == b"outside" + + +def test_corrupt_read_rejects_before_yielding(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + digest, _ = store.put([b"good"], max_bytes=10) + store._path(digest).write_bytes(b"bad") + with pytest.raises(BlobCorruptionError): + list(store.read(digest, max_bytes=10)) + + +def test_oversize_and_valid_digest_mismatch_leave_no_partial(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + with pytest.raises(BlobLimitError): + store.put([b"too-large"], max_bytes=1) + assert not list(store.temporary.glob("*.partial")) + with pytest.raises(BlobCorruptionError): + store.put([b"value"], max_bytes=100, expected_sha256="0" * 64) + assert not list(store.temporary.glob("*.partial")) + + +def test_blob_root_symlink_purge_does_not_inspect_outside(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + store.put([b"inside"], max_bytes=100) + outside = tmp_path / "outside" + outside.mkdir() + outside_file = outside / "outside-blob" + outside_file.write_bytes(b"outside") + for entry in store.blobs.iterdir(): + if entry.is_file(): + entry.unlink() + else: + for child in entry.iterdir(): + child.unlink() + entry.rmdir() + store.blobs.rmdir() + store.blobs.symlink_to(outside, target_is_directory=True) + assert store.purge_dry_run(set()) == [] + assert outside_file.read_bytes() == b"outside" + + +def test_purge_dry_run_does_not_follow_symlink(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + digest, _ = store.put([b"inside"], max_bytes=100) + outside = tmp_path / "outside" + outside.write_bytes(b"outside") + unsafe = store.blobs / "ff" + unsafe.symlink_to(tmp_path) + assert store.purge_dry_run(set()) == [store._path(digest).relative_to(store.root)] + assert outside.read_bytes() == b"outside" + + +def test_operation_lock_rejects_symlink_and_unsafe_mode(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + store.root.mkdir(mode=0o700) + outside = tmp_path / "outside" + outside.write_bytes(b"x") + store.lock_path.symlink_to(outside) + with pytest.raises(BlobLockError), store.operation_lock(): + pass + store.lock_path.unlink() + store.lock_path.write_bytes(b"") + store.lock_path.chmod(0o644) + with pytest.raises(BlobLockError), store.operation_lock(): + pass + + +def test_operation_lock_timeout_and_release_after_exception(tmp_path: Path) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + entered = threading.Event() + outcome: list[type[Exception]] = [] + + def contender() -> None: + entered.set() + try: + with store.operation_lock(timeout_seconds=0.02): + pass + except BlobLockError as error: + outcome.append(type(error)) + + with store.operation_lock(): + thread = threading.Thread(target=contender) + thread.start() + assert entered.wait(0.2) + thread.join(0.5) + assert not thread.is_alive() + assert outcome == [BlobLockError] + with pytest.raises(RuntimeError), store.operation_lock(): + raise RuntimeError("fault") + with store.operation_lock(timeout_seconds=0.02): + pass + for invalid in (True, "1", 0, 6): + with ( + pytest.raises(ValueError), + store.operation_lock( + timeout_seconds=invalid # type: ignore[arg-type] + ), + ): + pass + + +def test_expected_size_mismatch_rejects_before_blob_read( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + store = ContentAddressedBlobStore((tmp_path / "data").resolve()) + digest, _ = store.put([b"value"], max_bytes=100) + store._path(digest).write_bytes(b"x" * 4096) + reads = 0 + original = os.read + + def forbidden_read(fd: int, size: int) -> bytes: + nonlocal reads + reads += 1 + return original(fd, size) + + monkeypatch.setattr(os, "read", forbidden_read) + with pytest.raises(BlobCorruptionError): + store.verify(digest, expected_size=5) + assert reads == 0 diff --git a/tools/game-data-capture/tests/unit/test_cli.py b/tools/game-data-capture/tests/unit/test_cli.py new file mode 100644 index 0000000..593e4f7 --- /dev/null +++ b/tools/game-data-capture/tests/unit/test_cli.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import io +import logging +import subprocess +import sys +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +import pytest + +from game_data_capture.cli import EXIT_CONFIG_ERROR, EXIT_UNIMPLEMENTED, main +from game_data_capture.logging import RedactingFormatter, configure_logging, redact_log_data + + +def test_doctor_reports_required_diagnostics_without_openai_value( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("OPENAI_API_KEY", "sk-this-must-not-appear") + output = io.StringIO() + + with redirect_stdout(output): + result = main(["doctor", "--json"]) + + report = output.getvalue() + assert result == 0 + for name in ("niri", "wl-paste", "ffmpeg", "gpu_provider", "storage_permissions", "openai"): + assert f'"name":"{name}"' in report + assert "sk-this-must-not-appear" not in report + + +def test_serve_rejects_public_binding() -> None: + error = io.StringIO() + + with redirect_stderr(error): + result = main(["serve", "--host", "0.0.0.0"]) + + assert result == EXIT_CONFIG_ERROR + assert "loopback IP literal" in error.getvalue() + + +def test_serve_rejects_invalid_port_without_echoing_value() -> None: + error = io.StringIO() + + with redirect_stderr(error), pytest.raises(SystemExit) as raised: + main(["serve", "--port", "99999-private-value"]) + + assert raised.value.code == EXIT_CONFIG_ERROR + assert "port must be an integer from 1024 to 65535" in error.getvalue() + assert "99999-private-value" not in error.getvalue() + + +def test_subsystem_commands_are_explicit_stubs() -> None: + error = io.StringIO() + + with redirect_stderr(error): + result = main(["watch"]) + + assert result == EXIT_UNIMPLEMENTED + assert "requires gdc-runtime-integration" in error.getvalue() + + +def test_unimplemented_command_validates_config_before_returning_stub_status( + tmp_path: Path, +) -> None: + config_path = tmp_path / "invalid.toml" + config_path.write_text("[server]\nport = 80\n", encoding="utf-8") + error = io.StringIO() + + with redirect_stderr(error): + result = main(["watch", "--config", str(config_path)]) + + assert result == EXIT_CONFIG_ERROR + assert "server.port" in error.getvalue() + assert "subsystem is not implemented" not in error.getvalue() + + +def test_logging_redacts_sensitive_fields_and_messages() -> None: + redacted = redact_log_data( + { + "authorization": "Bearer secret-value", + "image_payload": "data:image/png;base64,abc", + "ocr_text": "unrestricted source text", + "status": "ok", + } + ) + record = logging.LogRecord( + "test", logging.INFO, __file__, 1, "Authorization: Bearer hidden sk-visible123", (), None + ) + + assert redacted == { + "authorization": "[REDACTED]", + "image_payload": "[REDACTED]", + "ocr_text": "[REDACTED]", + "status": "ok", + } + assert "hidden" not in RedactingFormatter("%(message)s").format(record) + assert "sk-visible123" not in RedactingFormatter("%(message)s").format(record) + + +def test_configure_logging_replaces_unredacted_handlers_and_redacts_exceptions() -> None: + package_logger = logging.getLogger("game_data_capture") + package_handlers = package_logger.handlers[:] + package_level = package_logger.level + package_propagate = package_logger.propagate + root_logger = logging.getLogger() + root_handler = logging.NullHandler() + root_logger.addHandler(root_handler) + leaked_output = io.StringIO() + existing_handler = logging.StreamHandler(leaked_output) + package_logger.handlers = [existing_handler] + captured_output = io.StringIO() + + try: + with redirect_stderr(captured_output): + configured_logger = configure_logging() + configure_logging() + try: + raise RuntimeError("Authorization: Bearer exception-secret; ocr_text=private text") + except RuntimeError: + configured_logger.exception("capture failed") + + output = captured_output.getvalue() + assert package_logger.handlers != [existing_handler] + assert len(package_logger.handlers) == 1 + assert leaked_output.getvalue() == "" + assert root_handler in root_logger.handlers + assert "exception-secret" not in output + assert "private text" not in output + assert "[REDACTED EXCEPTION]" in output + finally: + for handler in package_logger.handlers[:]: + package_logger.removeHandler(handler) + package_logger.handlers = package_handlers + package_logger.setLevel(package_level) + package_logger.propagate = package_propagate + root_logger.removeHandler(root_handler) + + +def test_package_import_has_no_runtime_side_effects(tmp_path: Path) -> None: + root = tmp_path / "fresh-xdg" + source_root = Path(__file__).parents[2] / "src" + environment = { + "GDC_IMPORT_ROOT": str(root), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": str(source_root), + } + script = """ +import os +import sys +from pathlib import Path + +root = Path(os.environ["GDC_IMPORT_ROOT"]) +targets = [root / name for name in ("config", "data", "cache", "runtime")] +events = [] + +def audit(event, arguments): + if event in {"os.mkdir", "os.chmod", "subprocess.Popen", "socket.connect", "sqlite3.connect"}: + events.append(event) + if event == "open": + mode = arguments[1] if len(arguments) > 1 else None + flags = arguments[2] if len(arguments) > 2 else 0 + if isinstance(mode, str) and any(marker in mode for marker in ("w", "a", "x", "+")): + events.append("open-write") + if isinstance(flags, int) and flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC): + events.append("open-write") + +sys.addaudithook(audit) +os.environ.update({ + "HOME": str(root / "home"), + "XDG_CONFIG_HOME": str(targets[0]), + "XDG_DATA_HOME": str(targets[1]), + "XDG_CACHE_HOME": str(targets[2]), + "XDG_RUNTIME_DIR": str(targets[3]), +}) +import game_data_capture + +if events or any(path.exists() for path in targets): + raise SystemExit(1) +""" + + completed = subprocess.run( + [sys.executable, "-B", "-c", script], + capture_output=True, + check=False, + env=environment, + text=True, + timeout=5, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tools/game-data-capture/tests/unit/test_config.py b/tools/game-data-capture/tests/unit/test_config.py new file mode 100644 index 0000000..784eb66 --- /dev/null +++ b/tools/game-data-capture/tests/unit/test_config.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path +from stat import S_IMODE + +import pytest + +from game_data_capture.config import ConfigError, ServerConfig, parse_config +from game_data_capture.paths import ( + PathConfigurationError, + prepare_private_directories, + prepare_private_file, + resolve_runtime_paths, +) + + +def test_config_uses_typed_defaults() -> None: + config = parse_config({}) + + assert config.server == ServerConfig(host="127.0.0.1", port=8765) + assert config.lifecycle.instance_name == "shd-capture" + assert config.limits.max_queue_items == 64 + + +@pytest.mark.parametrize("host", ["0.0.0.0", "192.168.1.2", "localhost", "example.test"]) +def test_config_rejects_non_loopback_binding(host: str) -> None: + with pytest.raises(ConfigError, match="loopback IP literal"): + parse_config({"server": {"host": host}}) + + +@pytest.mark.parametrize( + ("document", "message"), + [ + ({"server": {"port": 80}}, "server.port"), + ({"limits": {"max_queue_items": 257}}, "limits.max_queue_items"), + ({"lifecycle": {"instance_name": "SHD Capture"}}, "lifecycle.instance_name"), + ({"openai": {"api_key_env": "openai-key"}}, "openai.api_key_env"), + ], +) +def test_config_rejects_invalid_bounded_values(document: dict[str, object], message: str) -> None: + with pytest.raises(ConfigError, match=message): + parse_config(document) + + +def test_xdg_paths_are_resolved_without_writing(tmp_path: Path) -> None: + root = tmp_path / "xdg" + paths = resolve_runtime_paths( + { + "HOME": str(tmp_path / "home"), + "XDG_CONFIG_HOME": str(root / "config"), + "XDG_DATA_HOME": str(root / "data"), + "XDG_CACHE_HOME": str(root / "cache"), + "XDG_RUNTIME_DIR": str(root / "runtime"), + } + ) + + assert paths.runtime_dir == root / "runtime" / "shd-capture" + assert not root.exists() + + +def test_xdg_paths_reject_relative_runtime_path() -> None: + with pytest.raises(PathConfigurationError, match="XDG runtime"): + resolve_runtime_paths({"HOME": "/tmp/home", "XDG_RUNTIME_DIR": "relative"}) + + +def test_runtime_directories_and_lock_file_are_owner_only(tmp_path: Path) -> None: + paths = resolve_runtime_paths( + { + "HOME": str(tmp_path), + "XDG_CONFIG_HOME": str(tmp_path / "config"), + "XDG_DATA_HOME": str(tmp_path / "data"), + "XDG_CACHE_HOME": str(tmp_path / "cache"), + "XDG_RUNTIME_DIR": str(tmp_path / "runtime"), + } + ) + + prepare_private_directories(paths) + prepare_private_file(paths.instance_lock_file) + + assert S_IMODE(paths.runtime_dir.stat().st_mode) == 0o700 + assert S_IMODE(paths.instance_lock_file.stat().st_mode) == 0o600 diff --git a/tools/game-data-capture/tests/unit/web/test_app.py b/tools/game-data-capture/tests/unit/web/test_app.py new file mode 100644 index 0000000..565a133 --- /dev/null +++ b/tools/game-data-capture/tests/unit/web/test_app.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +from fastapi.testclient import TestClient +from starlette.routing import Mount +from starlette.staticfiles import StaticFiles +from starlette.types import Scope + +from game_data_capture.config import ServerConfig +from game_data_capture.web.app import ( + CaptureRecord, + CaptureSessionStatus, + ControlOutcome, + MediaDescriptor, + MutationOutcome, + WebConfig, + create_app, +) +from game_data_capture.web.security import new_bootstrap_token + +_CAPTURE_ID = "capture-00000000000000000000000000000001" +_LABEL: dict[str, object] = { + "family": "weapon", + "view": "detail", + "gear_slot": None, + "record_types": ["weapon", "weapon_talent"], + "confidence": 1.0, + "abstention_reason": None, +} + + +@dataclass +class FakeControl: + armed: bool = False + starts: int = 0 + stops: int = 0 + + def status(self) -> CaptureSessionStatus: + return CaptureSessionStatus(self.armed, 0, 64) + + def start_session(self) -> ControlOutcome: + if self.armed: + return "already_armed" + self.starts += 1 + self.armed = True + return "started" + + def stop_session(self) -> ControlOutcome: + if not self.armed: + return "already_stopped" + self.stops += 1 + self.armed = False + return "stopped" + + +@dataclass +class FakeCorpus: + records: dict[str, CaptureRecord] = field( + default_factory=lambda: { + _CAPTURE_ID: CaptureRecord(_CAPTURE_ID, "stored", "2026-07-31T00:00:00Z", None) + } + ) + calls: list[str] = field(default_factory=list) + media: MediaDescriptor | None = None + + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + self.calls.append("list") + return tuple(self.records.values())[:limit] + + def get_capture(self, capture_id: str) -> CaptureRecord | None: + self.calls.append("get") + return self.records.get(capture_id) + + def open_media(self, capture_id: str) -> MediaDescriptor | None: + self.calls.append("media") + if capture_id not in self.records: + return None + if self.media is not None: + return self.media + return MediaDescriptor("image/png", 4, [b"data"]) + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: + self.calls.append("label") + if capture_id not in self.records: + return "not_found" + old = self.records[capture_id] + self.records[capture_id] = CaptureRecord( + old.capture_id, old.state, old.received_at, dict(classification) + ) + return "updated" + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: + self.calls.append("reject") + return "updated" if capture_id in self.records else "not_found" + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: + self.calls.append("delete") + return "retained" if capture_id in self.records else "not_found" + + +def _client(control: FakeControl, corpus: FakeCorpus) -> tuple[TestClient, str]: + token = new_bootstrap_token() + app = create_app( + config=WebConfig(ServerConfig()), + capture_control=control, + corpus=corpus, + bootstrap_token=token, + ) + return TestClient(app, base_url="http://127.0.0.1:8765"), token + + +def _bootstrap(client: TestClient, token: str) -> str: + response = client.post( + "/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token} + ) + assert response.status_code == 200 + return str(response.json()["csrf_token"]) + + +def _mutation_headers(csrf: str) -> dict[str, str]: + return {"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf} + + +def test_factory_and_authenticated_gets_never_arm_capture() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + + assert control.starts == 0 + assert client.get("/health").json() == {"status": "ok"} + csrf = _bootstrap(client, token) + assert client.get("/api/session").status_code == 200 + assert client.get("/api/captures").status_code == 200 + assert client.get(f"/api/captures/{_CAPTURE_ID}").status_code == 200 + assert control.starts == 0 + assert csrf + + +def test_static_contract_is_package_local_and_does_not_accept_a_filesystem_root() -> None: + control = FakeControl() + corpus = FakeCorpus() + app = create_app( + config=WebConfig(ServerConfig()), + capture_control=control, + corpus=corpus, + bootstrap_token=new_bootstrap_token(), + ) + static_mount = next( + route for route in app.routes if isinstance(route, Mount) and route.path == "/static" + ) + + assert isinstance(static_mount.app, StaticFiles) + assert static_mount.app.follow_symlink is False + assert static_mount.app.directory is not None + assert Path(static_mount.app.directory).name == "static" + assert not Path( + static_mount.app.get_path( + cast(Scope, {"path": "/static/../../etc/passwd", "root_path": ""}) + ) + ).is_absolute() + + +def test_bootstrap_cookie_is_host_only_http_only_and_never_echoes_bootstrap_token() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + + response = client.post( + "/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token} + ) + + assert response.status_code == 200 + assert token not in response.text + cookie = response.headers["set-cookie"].lower() + assert "httponly" in cookie + assert "samesite=strict" in cookie + assert "domain=" not in cookie + assert token not in cookie + assert response.json().keys() == {"csrf_token"} + + +def test_start_is_explicit_csrf_protected_idempotent_and_rotates_session() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + csrf = _bootstrap(client, token) + old_session = str(client.cookies["capture_session"]) + + response = client.post( + "/api/session/start", headers=_mutation_headers(csrf), json={"confirmation": "START"} + ) + + assert response.status_code == 200 + assert response.json()["result"] == "started" + assert control.starts == 1 + next_csrf = str(response.json()["csrf_token"]) + stale = client.post( + "/api/session/start", headers=_mutation_headers(csrf), json={"confirmation": "START"} + ) + assert stale.status_code == 403 + assert control.starts == 1 + expired_session = client.post( + "/api/session/start", + headers={**_mutation_headers(next_csrf), "Cookie": f"capture_session={old_session}"}, + json={"confirmation": "START"}, + ) + assert expired_session.status_code == 401 + assert control.starts == 1 + duplicate = client.post( + "/api/session/start", headers=_mutation_headers(next_csrf), json={"confirmation": "START"} + ) + assert duplicate.status_code == 200 + assert duplicate.json()["result"] == "already_armed" + assert control.starts == 1 + + +def test_label_reject_and_retention_deletion_use_closed_confirmed_contracts() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + csrf = _bootstrap(client, token) + + labeled = client.post( + f"/api/captures/{_CAPTURE_ID}/label", headers=_mutation_headers(csrf), json=_LABEL + ) + assert labeled.status_code == 200 + csrf = str(labeled.json()["csrf_token"]) + detail = client.get(f"/api/captures/{_CAPTURE_ID}") + assert detail.json()["capture"]["classification"] == _LABEL + rejected = client.post( + f"/api/captures/{_CAPTURE_ID}/reject", + headers=_mutation_headers(csrf), + json={"confirmation": "REJECT"}, + ) + assert rejected.status_code == 200 + csrf = str(rejected.json()["csrf_token"]) + denied = client.post( + f"/api/captures/{_CAPTURE_ID}/delete", + headers=_mutation_headers(csrf), + json={"confirmation": "delete"}, + ) + assert denied.status_code == 403 + assert "delete" not in corpus.calls + retained = client.post( + f"/api/captures/{_CAPTURE_ID}/delete", + headers=_mutation_headers(csrf), + json={"confirmation": "DELETE"}, + ) + assert retained.status_code == 409 + assert retained.json() == {"error": "retained"} + assert corpus.calls.count("label") == 1 + assert corpus.calls.count("reject") == 1 + assert corpus.calls.count("delete") == 1 + + +def test_opaque_media_route_has_private_nosniff_response_and_no_path_parameter() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + _bootstrap(client, token) + + response = client.get(f"/api/captures/{_CAPTURE_ID}/media") + + assert response.status_code == 200 + assert response.content == b"data" + assert response.headers["content-type"] == "image/png" + assert response.headers["cache-control"] == "private, no-store" + assert response.headers["x-content-type-options"] == "nosniff" + assert client.get("/api/captures/../../etc/passwd/media").status_code == 404 + assert corpus.calls.count("media") == 1 + + +class RaisingChunks: + def __iter__(self) -> Iterator[bytes]: + raise RuntimeError("raw-secret /private/capture/path") + + +def test_malformed_media_streams_fail_before_success_headers_or_content() -> None: + descriptors = ( + MediaDescriptor("image/png", 4, [b"abc"]), + MediaDescriptor("image/png", 3, [b"abcd"]), + MediaDescriptor("image/png", 3, [b""]), + MediaDescriptor("image/png", 3, cast(Iterable[bytes], ["bad"])), + MediaDescriptor("image/png", 3, RaisingChunks()), + ) + for descriptor in descriptors: + control = FakeControl() + corpus = FakeCorpus(media=descriptor) + client, token = _client(control, corpus) + _bootstrap(client, token) + + response = client.get(f"/api/captures/{_CAPTURE_ID}/media") + + assert response.status_code == 503 + assert response.json() == {"error": "service_unavailable"} + assert "raw-secret" not in response.text + assert "/private/capture/path" not in response.text + + +class FailingCorpus(FakeCorpus): + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + raise RuntimeError("raw-secret /private/capture/path") + + +def test_port_exceptions_are_sanitized_without_an_uncaught_client_error() -> None: + control = FakeControl() + corpus = FailingCorpus() + client, token = _client(control, corpus) + _bootstrap(client, token) + + response = client.get("/api/captures") + + assert response.status_code == 503 + assert response.json() == {"error": "service_unavailable"} + assert "raw-secret" not in response.text + assert "/private/capture/path" not in response.text + + +def test_taxonomy_payload_rejects_unknown_and_ambiguous_values_before_corpus_mutation() -> None: + control = FakeControl() + corpus = FakeCorpus() + client, token = _client(control, corpus) + csrf = _bootstrap(client, token) + unsupported = dict(_LABEL) + unsupported["view"] = "not-a-view" + unsupported["record_types"] = ["gear"] + + response = client.post( + f"/api/captures/{_CAPTURE_ID}/label", headers=_mutation_headers(csrf), json=unsupported + ) + + assert response.status_code == 403 + assert "label" not in corpus.calls diff --git a/tools/game-data-capture/tests/unit/web/test_security.py b/tools/game-data-capture/tests/unit/web/test_security.py new file mode 100644 index 0000000..c4f77c2 --- /dev/null +++ b/tools/game-data-capture/tests/unit/web/test_security.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from game_data_capture.config import ServerConfig +from game_data_capture.web.app import ( + CaptureCorpusPort, + CaptureRecord, + CaptureSessionStatus, + ControlOutcome, + MediaDescriptor, + MutationOutcome, + WebConfig, + create_app, +) +from game_data_capture.web.security import ( + LocalAuthority, + LocalRequestLimitsMiddleware, + RequestLimits, + new_bootstrap_token, +) + + +@dataclass +class Control: + start_calls: int = 0 + + def status(self) -> CaptureSessionStatus: + return CaptureSessionStatus(False, 0, 1) + + def start_session(self) -> ControlOutcome: + self.start_calls += 1 + return "started" + + def stop_session(self) -> ControlOutcome: + return "already_stopped" + + +class Corpus(CaptureCorpusPort): + def list_captures(self, *, limit: int) -> Sequence[CaptureRecord]: + return () + + def get_capture(self, capture_id: str) -> CaptureRecord | None: + return None + + def open_media(self, capture_id: str) -> MediaDescriptor | None: + return None + + def label(self, capture_id: str, classification: Mapping[str, object]) -> MutationOutcome: + return "not_found" + + def reject_preserving_evidence(self, capture_id: str) -> MutationOutcome: + return "not_found" + + def delete_with_retention(self, capture_id: str) -> MutationOutcome: + return "not_found" + + +class Clock: + def __init__(self) -> None: + self.value = 0.0 + + def __call__(self) -> float: + return self.value + + +def _app( + control: Control, *, now: Callable[[], float] | None = None, max_body: int = 16 * 1024 +) -> tuple[FastAPI, str]: + token = new_bootstrap_token() + config = WebConfig(ServerConfig(), request_limits=RequestLimits(max_body_bytes=max_body)) + if now is None: + app = create_app( + config=config, + capture_control=control, + corpus=Corpus(), + bootstrap_token=token, + ) + else: + app = create_app( + config=config, + capture_control=control, + corpus=Corpus(), + bootstrap_token=token, + now=now, + ) + return app, token + + +def _bootstrap(client: TestClient, token: str) -> str: + result = client.post( + "/api/bootstrap", headers={"Origin": "http://127.0.0.1:8765"}, json={"token": token} + ) + assert result.status_code == 200 + return str(result.json()["csrf_token"]) + + +def test_host_origin_and_csrf_fail_before_start_side_effect() -> None: + control = Control() + app, token = _app(control) + client = TestClient(app, base_url="http://127.0.0.1:8765") + csrf = _bootstrap(client, token) + + bad_host = client.post( + "/api/session/start", + headers={"Host": "evil.test:8765", "Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf}, + json={"confirmation": "START"}, + ) + missing_origin = client.post( + "/api/session/start", headers={"X-CSRF-Token": csrf}, json={"confirmation": "START"} + ) + cross_origin = client.post( + "/api/session/start", + headers={"Origin": "http://evil.test", "X-CSRF-Token": csrf}, + json={"confirmation": "START"}, + ) + bad_csrf = client.post( + "/api/session/start", + headers={"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": "x" * 43}, + json={"confirmation": "START"}, + ) + + assert bad_host.status_code == 400 + assert missing_origin.status_code == 403 + assert cross_origin.status_code == 403 + assert bad_csrf.status_code == 403 + assert control.start_calls == 0 + + +def test_session_expiry_and_process_restart_invalidate_cookie() -> None: + control = Control() + clock = Clock() + app, token = _app(control, now=clock) + client = TestClient(app, base_url="http://127.0.0.1:8765") + _bootstrap(client, token) + assert client.get("/api/session").status_code == 200 + clock.value = 901 + assert client.get("/api/session").status_code == 401 + + new_app, _ = _app(control) + restarted = TestClient(new_app, base_url="http://127.0.0.1:8765") + restarted.cookies.update(client.cookies) + assert restarted.get("/api/session").status_code == 401 + + +def test_duplicate_json_and_form_payloads_fail_before_start_side_effect() -> None: + control = Control() + app, token = _app(control) + client = TestClient(app, base_url="http://127.0.0.1:8765") + csrf = _bootstrap(client, token) + headers = {"Origin": "http://127.0.0.1:8765", "X-CSRF-Token": csrf} + + duplicate_json = client.post( + "/api/session/start", + headers={**headers, "Content-Type": "application/json"}, + content=b'{"confirmation":"START","confirmation":"START"}', + ) + form = client.post( + "/api/session/start", + headers={**headers, "Content-Type": "application/x-www-form-urlencoded"}, + content=b"confirmation=START&confirmation=START", + ) + + assert duplicate_json.status_code == 403 + assert form.status_code == 403 + assert control.start_calls == 0 + + +def test_ipv6_host_requires_bracketed_exact_authority() -> None: + control = Control() + token = new_bootstrap_token() + app = create_app( + config=WebConfig(ServerConfig(host="::1", port=8765)), + capture_control=control, + corpus=Corpus(), + bootstrap_token=token, + ) + client = TestClient(app, base_url="http://127.0.0.1:8765") + + accepted = client.get("/health", headers={"Host": "[::1]:8765"}) + rejected = client.get("/health", headers={"Host": "::1:8765"}) + + assert accepted.status_code == 200 + assert rejected.status_code == 400 + + +def test_all_success_and_error_responses_have_local_only_security_headers() -> None: + control = Control() + app, _ = _app(control) + client = TestClient(app, base_url="http://127.0.0.1:8765") + responses = ( + client.get("/health"), + client.get("/health", headers={"Host": "evil.test:8765"}), + client.get("/api/session"), + ) + for response in responses: + csp = response.headers["content-security-policy"] + assert "default-src 'self'" in csp + assert "script-src 'self'" in csp + assert "style-src 'self'" in csp + assert "font-src 'self'" in csp + assert "img-src 'self'" in csp + assert "media-src 'self'" in csp + assert "connect-src 'self'" in csp + assert "object-src 'none'" in csp + assert "base-uri 'none'" in csp + assert "frame-ancestors 'none'" in csp + assert response.headers["referrer-policy"] == "no-referrer" + assert response.headers["x-content-type-options"] == "nosniff" + assert response.headers["x-frame-options"] == "DENY" + + +async def _call_raw(app: ASGIApp, scope: Scope, messages: list[Message]) -> list[Message]: + sent: list[Message] = [] + + async def receive() -> Message: + return messages.pop(0) + + async def send(message: Message) -> None: + sent.append(message) + + await app(scope, receive, send) + return sent + + +def _scope(path: str, headers: list[tuple[bytes, bytes]], *, method: str = "POST") -> Scope: + return { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": headers, + "client": ("127.0.0.1", 50000), + "server": ("127.0.0.1", 8765), + } + + +def test_declared_streamed_duplicate_and_disconnect_requests_fail_before_start() -> None: + control = Control() + app, _ = _app(control, max_body=32) + base_headers = [(b"host", b"127.0.0.1:8765"), (b"content-type", b"application/json")] + + declared = asyncio.run( + _call_raw( + app, + _scope("/api/session/start", [*base_headers, (b"content-length", b"33")]), + [{"type": "http.request", "body": b"", "more_body": False}], + ) + ) + duplicated = asyncio.run( + _call_raw( + app, + _scope("/api/session/start", [*base_headers, (b"host", b"127.0.0.1:8765")]), + [{"type": "http.request", "body": b"{}", "more_body": False}], + ) + ) + streamed = asyncio.run( + _call_raw( + app, + _scope( + "/api/bootstrap", + [*base_headers, (b"origin", b"http://127.0.0.1:8765")], + ), + [ + {"type": "http.request", "body": b'{"token":"', "more_body": True}, + {"type": "http.request", "body": b"x" * 24, "more_body": False}, + ], + ) + ) + disconnected = asyncio.run( + _call_raw( + app, + _scope( + "/api/bootstrap", + [*base_headers, (b"origin", b"http://127.0.0.1:8765")], + ), + [{"type": "http.disconnect"}], + ) + ) + + assert declared[0]["status"] == 413 + assert duplicated[0]["status"] == 400 + assert streamed[0]["status"] == 413 + assert disconnected[0]["status"] == 400 + assert control.start_calls == 0 + + +def test_post_completion_disconnect_is_normal_for_streaming_response_lifecycle() -> None: + async def streaming_lifecycle(scope: Scope, receive: Receive, send: Send) -> None: + await receive() + assert (await receive())["type"] == "http.disconnect" + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + guarded = LocalRequestLimitsMiddleware( + streaming_lifecycle, + authority=LocalAuthority("127.0.0.1", 8765), + limits=RequestLimits(), + ) + sent = asyncio.run( + _call_raw( + guarded, + _scope("/stream", [(b"host", b"127.0.0.1:8765")], method="GET"), + [ + {"type": "http.request", "body": b"", "more_body": False}, + {"type": "http.disconnect"}, + ], + ) + ) + + assert sent[0]["status"] == 200 diff --git a/tools/game-data-capture/uv.lock b/tools/game-data-capture/uv.lock new file mode 100644 index 0000000..dd22b74 --- /dev/null +++ b/tools/game-data-capture/uv.lock @@ -0,0 +1,465 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "fastapi" +version = "0.115.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/55/ae499352d82338331ca1e28c7f4a63bfd09479b16395dce38cf50a39e2c2/fastapi-0.115.12.tar.gz", hash = "sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681", size = 295236, upload-time = "2025-03-23T22:55:43.822Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/b3/b51f09c2ba432a576fe63758bddc81f78f0c6309d9e5c10d194313bf021e/fastapi-0.115.12-py3-none-any.whl", hash = "sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d", size = 95164, upload-time = "2025-03-23T22:55:42.101Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + +[[package]] +name = "mypy" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/43/d5e49a86afa64bd3839ea0d5b9c7103487007d728e1293f52525d6d5486a/mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43", size = 3239717, upload-time = "2025-02-05T03:50:34.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/3a/03c74331c5eb8bd025734e04c9840532226775c47a2c39b56a0c8d4f128d/mypy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:aea39e0583d05124836ea645f412e88a5c7d0fd77a6d694b60d9b6b2d9f184fd", size = 10793981, upload-time = "2025-02-05T03:50:28.25Z" }, + { url = "https://files.pythonhosted.org/packages/f0/1a/41759b18f2cfd568848a37c89030aeb03534411eef981df621d8fad08a1d/mypy-1.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f2147ab812b75e5b5499b01ade1f4a81489a147c01585cda36019102538615f", size = 9749175, upload-time = "2025-02-05T03:50:13.411Z" }, + { url = "https://files.pythonhosted.org/packages/12/7e/873481abf1ef112c582db832740f4c11b2bfa510e829d6da29b0ab8c3f9c/mypy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce436f4c6d218a070048ed6a44c0bbb10cd2cc5e272b29e7845f6a2f57ee4464", size = 11455675, upload-time = "2025-02-05T03:50:31.421Z" }, + { url = "https://files.pythonhosted.org/packages/b3/d0/92ae4cde706923a2d3f2d6c39629134063ff64b9dedca9c1388363da072d/mypy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8023ff13985661b50a5928fc7a5ca15f3d1affb41e5f0a9952cb68ef090b31ee", size = 12410020, upload-time = "2025-02-05T03:48:48.705Z" }, + { url = "https://files.pythonhosted.org/packages/46/8b/df49974b337cce35f828ba6fda228152d6db45fed4c86ba56ffe442434fd/mypy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e", size = 12498582, upload-time = "2025-02-05T03:49:03.628Z" }, + { url = "https://files.pythonhosted.org/packages/13/50/da5203fcf6c53044a0b699939f31075c45ae8a4cadf538a9069b165c1050/mypy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22", size = 9366614, upload-time = "2025-02-05T03:50:00.313Z" }, + { url = "https://files.pythonhosted.org/packages/09/4e/a7d65c7322c510de2c409ff3828b03354a7c43f5a8ed458a7a131b41c7b9/mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e", size = 2221777, upload-time = "2025-02-05T03:50:08.348Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pytest" +version = "8.3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload-time = "2025-03-02T12:54:52.069Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158, upload-time = "2024-12-16T19:45:46.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, +] + +[[package]] +name = "ruff" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/2b/7ca27e854d92df5e681e6527dc0f9254c9dc06c8408317893cf96c851cdd/ruff-0.11.0.tar.gz", hash = "sha256:e55c620690a4a7ee6f1cccb256ec2157dc597d109400ae75bbf944fc9d6462e2", size = 3799407, upload-time = "2025-03-14T13:52:36.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/40/3d0340a9e5edc77d37852c0cd98c5985a5a8081fc3befaeb2ae90aaafd2b/ruff-0.11.0-py3-none-linux_armv6l.whl", hash = "sha256:dc67e32bc3b29557513eb7eeabb23efdb25753684b913bebb8a0c62495095acb", size = 10098158, upload-time = "2025-03-14T13:51:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/ec/a9/d8f5abb3b87b973b007649ac7bf63665a05b2ae2b2af39217b09f52abbbf/ruff-0.11.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:38c23fd9bdec4eb437b4c1e3595905a0a8edfccd63a790f818b28c78fe345639", size = 10879071, upload-time = "2025-03-14T13:51:58.989Z" }, + { url = "https://files.pythonhosted.org/packages/ab/62/aaa198614c6211677913ec480415c5e6509586d7b796356cec73a2f8a3e6/ruff-0.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7c8661b0be91a38bd56db593e9331beaf9064a79028adee2d5f392674bbc5e88", size = 10247944, upload-time = "2025-03-14T13:52:02.318Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/59e0a9f2cf1ce5e6cbe336b6dd0144725c8ea3b97cac60688f4e7880bf13/ruff-0.11.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6c0e8d3d2db7e9f6efd884f44b8dc542d5b6b590fc4bb334fdbc624d93a29a2", size = 10421725, upload-time = "2025-03-14T13:52:04.303Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c3/dcd71acc6dff72ce66d13f4be5bca1dbed4db678dff2f0f6f307b04e5c02/ruff-0.11.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c3156d3f4b42e57247275a0a7e15a851c165a4fc89c5e8fa30ea6da4f7407b8", size = 9954435, upload-time = "2025-03-14T13:52:06.602Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9a/342d336c7c52dbd136dee97d4c7797e66c3f92df804f8f3b30da59b92e9c/ruff-0.11.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:490b1e147c1260545f6d041c4092483e3f6d8eba81dc2875eaebcf9140b53905", size = 11492664, upload-time = "2025-03-14T13:52:08.613Z" }, + { url = "https://files.pythonhosted.org/packages/84/35/6e7defd2d7ca95cc385ac1bd9f7f2e4a61b9cc35d60a263aebc8e590c462/ruff-0.11.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1bc09a7419e09662983b1312f6fa5dab829d6ab5d11f18c3760be7ca521c9329", size = 12207856, upload-time = "2025-03-14T13:52:11.019Z" }, + { url = "https://files.pythonhosted.org/packages/22/78/da669c8731bacf40001c880ada6d31bcfb81f89cc996230c3b80d319993e/ruff-0.11.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcfa478daf61ac8002214eb2ca5f3e9365048506a9d52b11bea3ecea822bb844", size = 11645156, upload-time = "2025-03-14T13:52:13.383Z" }, + { url = "https://files.pythonhosted.org/packages/ee/47/e27d17d83530a208f4a9ab2e94f758574a04c51e492aa58f91a3ed7cbbcb/ruff-0.11.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fbb2aed66fe742a6a3a0075ed467a459b7cedc5ae01008340075909d819df1e", size = 13884167, upload-time = "2025-03-14T13:52:15.528Z" }, + { url = "https://files.pythonhosted.org/packages/9f/5e/42ffbb0a5d4b07bbc642b7d58357b4e19a0f4774275ca6ca7d1f7b5452cd/ruff-0.11.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92c0c1ff014351c0b0cdfdb1e35fa83b780f1e065667167bb9502d47ca41e6db", size = 11348311, upload-time = "2025-03-14T13:52:18.088Z" }, + { url = "https://files.pythonhosted.org/packages/c8/51/dc3ce0c5ce1a586727a3444a32f98b83ba99599bb1ebca29d9302886e87f/ruff-0.11.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e4fd5ff5de5f83e0458a138e8a869c7c5e907541aec32b707f57cf9a5e124445", size = 10305039, upload-time = "2025-03-14T13:52:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/60/e0/475f0c2f26280f46f2d6d1df1ba96b3399e0234cf368cc4c88e6ad10dcd9/ruff-0.11.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:96bc89a5c5fd21a04939773f9e0e276308be0935de06845110f43fd5c2e4ead7", size = 9937939, upload-time = "2025-03-14T13:52:22.798Z" }, + { url = "https://files.pythonhosted.org/packages/e2/d3/3e61b7fd3e9cdd1e5b8c7ac188bec12975c824e51c5cd3d64caf81b0331e/ruff-0.11.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a9352b9d767889ec5df1483f94870564e8102d4d7e99da52ebf564b882cdc2c7", size = 10923259, upload-time = "2025-03-14T13:52:24.89Z" }, + { url = "https://files.pythonhosted.org/packages/30/32/cd74149ebb40b62ddd14bd2d1842149aeb7f74191fb0f49bd45c76909ff2/ruff-0.11.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:049a191969a10897fe052ef9cc7491b3ef6de79acd7790af7d7897b7a9bfbcb6", size = 11406212, upload-time = "2025-03-14T13:52:27.493Z" }, + { url = "https://files.pythonhosted.org/packages/00/ef/033022a6b104be32e899b00de704d7c6d1723a54d4c9e09d147368f14b62/ruff-0.11.0-py3-none-win32.whl", hash = "sha256:3191e9116b6b5bbe187447656f0c8526f0d36b6fd89ad78ccaad6bdc2fad7df2", size = 10310905, upload-time = "2025-03-14T13:52:30.46Z" }, + { url = "https://files.pythonhosted.org/packages/ed/8a/163f2e78c37757d035bd56cd60c8d96312904ca4a6deeab8442d7b3cbf89/ruff-0.11.0-py3-none-win_amd64.whl", hash = "sha256:c58bfa00e740ca0a6c43d41fb004cd22d165302f360aaa56f7126d544db31a21", size = 11411730, upload-time = "2025-03-14T13:52:32.508Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f7/096f6efabe69b49d7ca61052fc70289c05d8d35735c137ef5ba5ef423662/ruff-0.11.0-py3-none-win_arm64.whl", hash = "sha256:868364fc23f5aa122b00c6f794211e85f7e78f5dffdf7c590ab90b8c4e69b657", size = 10538956, upload-time = "2025-03-14T13:52:34.491Z" }, +] + +[[package]] +name = "shd-game-data-capture" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "fastapi" }, + { name = "jinja2" }, + { name = "python-multipart" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "jsonschema" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "types-jsonschema" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = "==0.115.12" }, + { name = "jinja2", specifier = "==3.1.6" }, + { name = "python-multipart", specifier = "==0.0.20" }, + { name = "uvicorn", specifier = "==0.34.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = "==0.28.1" }, + { name = "jsonschema", specifier = "==4.25.1" }, + { name = "mypy", specifier = "==1.15.0" }, + { name = "pytest", specifier = "==8.3.5" }, + { name = "ruff", specifier = "==0.11.0" }, + { name = "types-jsonschema", specifier = "==4.25.1.20251009" }, +] + +[[package]] +name = "starlette" +version = "0.46.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/20/08dfcd9c983f6a6f4a1000d934b9e6d626cff8d2eeb77a89a68eef20a2b7/starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5", size = 2580846, upload-time = "2025-04-13T13:56:17.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload-time = "2025-04-13T13:56:16.21Z" }, +] + +[[package]] +name = "types-jsonschema" +version = "4.25.1.20251009" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/da/5b901088da5f710690b422137e8ae74197fb1ca471e4aa84dd3ef0d6e295/types_jsonschema-4.25.1.20251009.tar.gz", hash = "sha256:75d0f5c5dd18dc23b664437a0c1a625743e8d2e665ceaf3aecb29841f3a5f97f", size = 15661, upload-time = "2025-10-09T02:54:36.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/6a/e5146754c0dfc272f176db9c245bc43cc19030262d891a5a85d472797e60/types_jsonschema-4.25.1.20251009-py3-none-any.whl", hash = "sha256:f30b329037b78e7a60146b1146feb0b6fb0b71628637584409bada83968dad3e", size = 15925, upload-time = "2025-10-09T02:54:35.847Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568, upload-time = "2024-12-15T13:33:30.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", size = 62315, upload-time = "2024-12-15T13:33:27.467Z" }, +]