diff --git a/.env.example b/.env.example index 1be3073..cd973a3 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,8 @@ CONVEX_SELF_HOSTED_URL=http://localhost:3210 CONVEX_SELF_HOSTED_ADMIN_KEY= # Generate independent bridge/encryption/signing values using `bun run keys:generate`. CONVEX_BRIDGE_SECRET= +# Dedicated worker lifecycle credential; never share the bridge secret. +CONVEX_SYNC_WORKER_SECRET= TOKEN_ENCRYPTION_KEY= APP_AUTH_PRIVATE_KEY= APP_AUTH_JWKS= diff --git a/Dockerfile b/Dockerfile index c987727..6fe472e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ COPY package.json bun.lock ./ RUN bun install --frozen-lockfile --ignore-scripts COPY . . RUN bun run build +RUN bun build workers/sync.ts --target bun --outfile build/sync-worker.js FROM oven/bun@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS runtime ARG APP_RELEASE_ID=local @@ -17,6 +18,7 @@ RUN bun install --frozen-lockfile --production COPY --from=build /app/build ./build COPY --from=build /app/src/lib ./src/lib COPY --from=build /app/workers ./workers +COPY --from=build /app/scripts/run-sync-worker.ts ./scripts/run-sync-worker.ts COPY --from=build /app/fixtures ./fixtures RUN mkdir -p /app/data/reference && chown -R bun:bun /app/data USER bun diff --git a/README.md b/README.md index a5771b9..ce2e62d 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ Use [the live deployment guide](docs/live-setup.md) for `dev.evebot.app`, the re bunx convex dev --once ``` - `APP_AUTH_ISSUER`, `APP_AUTH_JWKS` and `CONVEX_BRIDGE_SECRET` must be set in **Convex's environment**, not merely the Docker process environment. The configure script handles these three. Custom JWT audience is `eve-intelligence`; the public JWKS data URI avoids a boot dependency on the web server. + `APP_AUTH_ISSUER`, `APP_AUTH_JWKS`, `CONVEX_BRIDGE_SECRET` and `CONVEX_SYNC_WORKER_SECRET` must be set in **Convex's environment**, not merely the Docker process environment. The direct configure command reads the local private `.env`, including `CONVEX_SELF_HOSTED_ADMIN_KEY`; never put that admin key in a web or worker environment. Custom JWT audience is `eve-intelligence`; the public JWKS data URI avoids a boot dependency on the web server. 6. Register an EVE application and exact callback at . Set `EVE_CLIENT_ID`, `EVE_CALLBACK_URL`, and `EVE_CLIENT_SECRET` for a confidential web client. A public PKCE client leaves the secret blank. Initial login requests `esi-assets.read_assets.v1`; later dataset upgrades preserve only server-allowlisted existing scopes. 7. Set `ESI_USER_AGENT` with your actual operator contact. Optional `TYPESAFE_API_KEY` enables server-side JEV routing. Set public/internal Convex URLs as described in `.env.example`. 8. Build the fixture reference database and start `bun run dev`. Connect a character, synchronize it, and add more characters with **Connect EVE Character**. Each connects explicitly to the current application user. +9. In a second terminal, create a private `.env.worker` containing only `CONVEX_URL`, `CONVEX_SYNC_WORKER_SECRET`, `TOKEN_ENCRYPTION_KEY`, `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET`, `EVE_CALLBACK_URL`, `ESI_USER_AGENT`, and worker tuning values, then start the worker with `bun --env-file=.env.worker run sync:worker`. Do not source the full web `.env`; the worker launcher intentionally rejects bridge/admin/JWT/JEV credentials. **Important:** the fixture DB contains only a handful of types. Live assets absent from it remain explicitly unpriced/unknown. Do not describe a fixture-backed report as a complete real-character valuation. See [full-import handoff](docs/reference-operations.md) before ingesting real releases. @@ -60,10 +61,11 @@ Compose uses Convex's default SQLite backend; it introduces no other datastore. - `backend`: persistent Convex SQLite volume, API bound to loopback by default (the configured Pangolin deployment explicitly overrides this). - `web`: Bun web service with read-only reference volume. +- `sync-worker`: dedicated durable asset-sync worker with a bundled runtime and worker-only secret. - `reference-worker`: dedicated one-shot heavy-ingestion process (`tools` profile). - `dashboard`: optional loopback-only management UI (`admin` profile). -For local container use, set `APP_ORIGIN` and `APP_AUTH_ISSUER` to `http://localhost:3000` and the EVE callback to `http://localhost:3000/auth/eve/callback`. Repeat `convex:configure` after changing issuer. +For local container use, set `APP_ORIGIN` and `APP_AUTH_ISSUER` to `http://localhost:3000` and the EVE callback to `http://localhost:3000/auth/eve/callback`. Repeat `bun run convex:configure` after changing issuer. ```sh docker compose up -d backend @@ -71,6 +73,8 @@ docker compose up -d backend # Explicit production archive import (~109 MiB for the reviewed releases): docker compose --profile tools run --rm --build reference-worker docker compose up -d --build web +# Start the explicitly profiled durable asset worker; Compose forwards only its worker environment: +docker compose --profile worker up -d --build sync-worker # Optional management UI: docker compose --profile admin up -d dashboard ``` @@ -109,9 +113,9 @@ docs/research/ current official-source evidence and captured ESI con ## Operational limits -- The configured development deployment has passed live TypeSafe routing and public Convex WebSocket checks; the operator has confirmed real EVE sign-in, asset synchronization, and matching Assets/chat valuation totals for one character. Live multi-character linking and revocation/transfer scenarios remain separate checks. +- This checkpoint records local/disposable prototype verification only. It makes no claim of deployed/live ESI verification or production readiness; live multi-character linking, revocation and transfer scenarios remain deferred. - Production imports use reviewed source adapters and immutable SQLite artifacts. CCP NPC station IDs, system/type/owner/position facts are imported when `npcStations.jsonl` is present; station names remain optional and the production profile explicitly marks station coverage incomplete until an isolated measured replay. Twenty-one dangling blueprint references in the reviewed release are preserved explicitly rather than fabricated. See reference operations and the verification record for executed import checks and limitations. -- Asset synchronization runs on demand in a single Bun web process. Convex leases/CAS protect staging and token rotation, but durable background character-sync job recovery and scheduled refresh are future work. +- Asset synchronization is enqueued by the web process and completed by the separate durable `sync-worker`. Convex leases/CAS protect staging and token rotation; start the worker explicitly with the worker-scoped launcher before testing live synchronization. - ESI page collections are not transactional snapshots. Stability checks reduce risk; cache rollover/mutation may require a later retry. - Location IDs are shown explicitly as unresolved. Fitted/nested assets are distinct ESI items; no contents multiplier is applied to ship prices. - Conversations/execution summaries persist for authenticated live users; this first UI does not yet provide a history browser. A pending guest request is kept only in tab `sessionStorage` and rechecked after login/sync. diff --git a/compose.yaml b/compose.yaml index aab4481..532995a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -54,6 +54,25 @@ services: volumes: ["reference-data:/app/data/reference:ro"] depends_on: backend: { condition: service_healthy } + sync-worker: + profiles: ["worker"] + restart: unless-stopped + build: . + command: ["bun", "scripts/run-sync-worker.ts"] + environment: + CONVEX_URL: http://backend:3210 + CONVEX_SYNC_WORKER_SECRET: ${CONVEX_SYNC_WORKER_SECRET:-} + TOKEN_ENCRYPTION_KEY: ${TOKEN_ENCRYPTION_KEY:-} + EVE_CLIENT_ID: ${EVE_CLIENT_ID:-} + EVE_CLIENT_SECRET: ${EVE_CLIENT_SECRET:-} + EVE_CALLBACK_URL: ${EVE_CALLBACK_URL:-http://localhost/unused-worker-callback} + ESI_USER_AGENT: ${ESI_USER_AGENT:-EVE-Intelligence/0.1 (configure operator contact)} + SYNC_WORKER_ID: ${SYNC_WORKER_ID:-local-sync-worker} + SYNC_WORKER_LEASE_MS: ${SYNC_WORKER_LEASE_MS:-600000} + SYNC_WORKER_HEARTBEAT_MS: ${SYNC_WORKER_HEARTBEAT_MS:-60000} + SYNC_WORKER_POLL_MS: ${SYNC_WORKER_POLL_MS:-1000} + depends_on: + backend: { condition: service_healthy } reference-worker: profiles: ["tools"] build: . diff --git a/docs/architecture.md b/docs/architecture.md index cd6bf8b..dc4e3d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,7 +30,7 @@ Dedicated Bun reference worker EVE authorization-code + PKCE and encrypted token storage runs in Bun. First login creates an app user keyed to a verified character and owner claim; explicit linking attaches additional characters to the current user. Conflicting ownership never silently merges accounts. Session cookies carry random opaque values; only hashes are stored. Application RS256 JWTs are short-lived and audience-scoped for the Convex browser client. EVE access/refresh tokens are never browser credentials. -Asset sync stages bounded chunks, then atomically changes a character's active generation after all pages succeed. A failed fetch preserves the previous complete generation. Sync metadata records freshness/error, and the domain gate refuses to present missing/stale coverage as complete. ESI page caches are isolated by authorization identity; no partial paginated result is accepted as a full snapshot. Single-host deployment initially runs one web process; token refresh also uses durable compare-and-swap to prevent a stale refresh from overwriting a rotated token. +Asset sync is a durable `asset_sync` job owned by the authenticated user; `/api/sync` only enqueues a server-derived, deduplicated request. A dedicated Bun worker claims bounded due work with renewable tokenized leases, stages validated chunks, then atomically changes a character's active generation after all pages succeed. A failed fetch preserves the previous complete generation. Sync metadata records freshness/error, and the domain gate refuses to present missing/stale coverage as complete. ESI page caches are isolated by authorization identity; no partial paginated result is accepted as a full snapshot. Cooldowns are persisted in Convex and shared by workers. Token refresh is coordinated by a durable per-character lease and compare-and-swap; uncertain rotating-token results fail closed as `reconnect_required` instead of replaying a refresh. The worker receives no application signing, JEV, reference, or deployment-admin secrets. This is a local T07 prototype; it requires `CONVEX_SYNC_WORKER_SECRET` and the explicit worker-scoped startup path. T08 location/containment enrichment remains feature-local, with integration at the post-assets/pre-publication seam still pending. ## Valuation diff --git a/docs/design-records/t07-durable-sync.md b/docs/design-records/t07-durable-sync.md new file mode 100644 index 0000000..e01956f --- /dev/null +++ b/docs/design-records/t07-durable-sync.md @@ -0,0 +1,84 @@ +# T07 durable asset synchronization + +Status: prototype implementation. This record describes the local, disposable +queue and does not claim production readiness. + +## Boundaries + +`POST /api/sync` authenticates the application session and only enqueues +server-derived asset jobs. It does not refresh OAuth credentials or call ESI. +Partial enqueue results carry per-character safe skip reasons; an all-failed +queue response is not reported as `202`. The assets and chat surfaces poll +`/api/sync/status?job=`, expose per-character queued/terminal state and +cancellation, and rerun the report only after every queued job succeeds. +`POST /api/sync/cancel` fences a job for its owning user. The Convex bridge +checks the session proof, character ownership, active grant and exact dataset +scope again when enqueueing and claiming. + +Jobs are user-owned `asset_sync` rows. The server derives the dedupe key +`user:character:assets`; only `queued` and `running` rows deduplicate. A claim +captures grant revision, authorization epoch, credential version and scopes. +Claims have random lease tokens and a ten-minute default lease, renewed every +minute by the worker. Expired claims are recovered by the next claim +transaction. Selection is bounded to 100 due rows and prefers users with no +running work before due/creation order, providing per-user fairness without a +second scheduler. + +Prototype budgets are conservative and explicit (worker lease/poll values are environment-configurable): five attempts per job, +250 assets per staged chunk, 4,096 chunks, 50,000 ESI rows, at most eight +active jobs per user, a ten-minute sync lease, and a 24-hour maximum retry +delay. ESI request/page/response limits are +owned by the T06 transport. Queue backpressure is visible as a bounded failed +job, never a silently truncated generation. + +## Fencing and delivery + +A worker starts a generation only through the job lease, stages through a +job-token mutation, and commits success or failure through a mutation that +rechecks the lease, job fence, character authorization and generation fence. +Success and failure therefore cannot mutate state after cancellation, scope +loss, credential rotation, quarantine or lease expiry. Staging is contiguous; +identical duplicate chunk delivery is idempotent while a changed duplicate is +rejected. A successful commit is idempotent if a response is lost. Failed +snapshots preserve the previous complete generation. + +`isSingleton` is validated by the T06 ESI parser and is carried unchanged in +every stored asset row; absent legacy values remain unknown. + +## Refresh coordination + +Refresh leases are durable per user/character rows, not process maps. The +worker decrypts a grant only inside the dedicated worker, performs exactly one +remote rotating-token refresh, then commits the replacement with the expected +credential version. The committed credential version/revision/epoch are the +only authority values used for ESI and publication. A lost, timed-out, or +otherwise uncertain refresh result is never replayed: the character becomes +`reconnect_required`, credentials are removed, active work is quarantined, and +all private datasets are blocked until explicit reconnect. + +The worker receives only its dedicated `CONVEX_SYNC_WORKER_SECRET`, +token-encryption, EVE OAuth and ESI user-agent credentials. Worker mutations do +not accept the general `CONVEX_BRIDGE_SECRET`; login/session/tenant operations +remain on the web-only bridge. It does not receive application JWT signing +keys, JEV keys, reference paths/artifacts, or deployment-admin credentials. The +web process no longer performs request-bound refresh or asset paging. The image +build bundles `workers/sync.ts` to `build/sync-worker.js`, so the runtime never +executes a raw `$lib`-aliased TypeScript entry. Compose starts that artifact +through `scripts/run-sync-worker.ts`; the launcher rejects bridge, admin, +JWT-signing and JEV credentials even when `bun run sync:worker` is invoked +directly. Start it with `bun run secrets:run --worker -- docker compose --profile worker up -d +--build sync-worker`, never with the general web/deployment secret modes. The Compose +service is explicitly profiled so plain `docker compose up` cannot launch a worker +with empty credentials. + +## Cooldowns and recovery + +Provider cooldowns are persisted in `syncCooldowns`, keyed by character +context and shared provider group. Asset jobs carry the provider group, and +claim selection plus the worker both honor the group's eligible-at timestamp +before ESI work. ESI remains the single transport retry/cooldown +implementation. No public-feed scheduler is introduced by this task. + +Retention, migrations, production quotas, remote revocation, reference +extraction and public market orchestration remain deferred as required by the +parallel-worktree plan. diff --git a/docs/live-setup.md b/docs/live-setup.md index bd1fbbe..80daa67 100644 --- a/docs/live-setup.md +++ b/docs/live-setup.md @@ -37,7 +37,7 @@ bun run secrets:status bun run secrets:run -- bun scripts/check-live-config.ts ``` -Expected application keys: `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET` for confidential SSO, `TYPESAFE_API_KEY`, `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `APP_AUTH_PRIVATE_KEY`, and `APP_AUTH_JWKS`. Generate the last four **once** using `bun run keys:generate`; import `.env.keys` into the selected Infisical environment without replacing existing API keys. The generator refuses to overwrite an existing file. After verifying the import and key pairing, remove the temporary copy. Back up the vault: changing/loss of the encryption key makes stored EVE credentials unreadable. Do not regenerate keys casually during deployment. +Expected application keys: `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET` for confidential SSO, `TYPESAFE_API_KEY`, `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `CONVEX_SYNC_WORKER_SECRET`, `APP_AUTH_PRIVATE_KEY`, and `APP_AUTH_JWKS`. Generate the key material **once** using `bun run keys:generate`; import `.env.keys` into the selected Infisical environment without replacing existing API keys. Configure both distinct bridge secrets in Convex with `bun run secrets:run --deployment --configure -- bun run convex:configure`; the sync worker receives only `CONVEX_SYNC_WORKER_SECRET`, while web receives only `CONVEX_BRIDGE_SECRET`. Use `bun run secrets:run --worker -- ...` for worker commands so session/JEV signing and general bridge secrets are not injected. The generator refuses to overwrite an existing file. After verifying the import and key pairing, remove the temporary copy. Back up the vault: changing/loss of the encryption key makes stored EVE credentials unreadable. Do not regenerate keys casually during deployment. The helper is an operator launcher, not a sandbox. Commands it launches can read injected secrets: never run `env`, `printenv`, shell tracing, or unrestricted `docker compose config` through it. Use `docker compose config --quiet` for validation. Docker administrators can inspect container environments; Infisical startup injection does not conceal secrets from host administrators. Updating a vault value requires recreating the affected container with fresh injected values. Encryption/signing-key rotation needs a separate migration plan. @@ -47,7 +47,7 @@ Run from the repository root: ```sh bun run secrets:run -- docker compose up -d backend -bun run secrets:run --deployment -- bun run convex:configure +bun run secrets:run --deployment --configure -- bun run convex:configure bun run secrets:run --deployment -- bun node_modules/convex/bin/main.js dev --once --typecheck disable ``` @@ -61,9 +61,10 @@ Use the production import procedure in [reference operations](reference-operatio ```sh bun run secrets:run -- docker compose up -d --build web +bun run secrets:run --worker -- docker compose --profile worker up -d --build sync-worker ``` -The container listens on 3000 internally but publishes **5173** on the host. This is the production SvelteKit/Bun server, not the Vite development server. The web mounts reference data read-only. The one-shot reference worker has the writable volume mount. +The web container listens on 3000 internally but publishes **5173** on the host. This is the production SvelteKit/Bun server, not the Vite development server. The web mounts reference data read-only. The dedicated sync-worker is a separate long-running service: its image starts the bundled `build/sync-worker.js` through the restricted launcher and receives only worker-safe secrets. The one-shot reference worker has the writable volume mount. ## First live verification diff --git a/docs/parallel-worktrees.md b/docs/parallel-worktrees.md index ea7bd0c..ad66aac 100644 --- a/docs/parallel-worktrees.md +++ b/docs/parallel-worktrees.md @@ -8,17 +8,20 @@ **Planning baseline:** `fdcc486` (`Initial Commit`). The roadmap records an earlier 97-test gate; each prototype change reruns the current local suite. Existing `jobs`, `userPreferences` and `referenceSyncMetadata` tables are placeholders, not services. -**T06 evidence:** The centralized ESI client now has fixed typed asset, asset-name, +**T06/T07 evidence:** The centralized ESI client has fixed typed asset, asset-name, asset-location, wallet-balance, and market-read strategies with bounded response bytes/rows/pages, bounded retries, abort-first cleanup, validators, canonical int64 helper bodies, complete scoped authority-context cache identity, cooldown queries, -and runtime schemas. Asset snapshots report bounded non-atomic coverage; page-one -revalidation is not presented as a transaction. T07 durable scheduling/worker -orchestration, T08 location/service orchestration, and T13 wallet persistence/UI/ -service orchestration remain unimplemented; production-readiness gates remain -deferred. The narrow `beginSync` expected-authority fence is part of T06's atomic -bridge contract: it prevents transport cache authority from being published under -a different T05 grant/revision/credential fence; it is not T07 durable worker orchestration. +and runtime schemas. T07 durable scheduling/worker orchestration is implemented as +a local prototype: the current full suite records 204 passing tests, including the +bundled dedicated worker, durable leases, fencing, cancellation/recovery, refresh +coordination and restricted startup. T08 location/service orchestration and T13 +wallet persistence/UI/service orchestration remain pending; T07/T08 integration is +not complete and production-readiness gates remain deferred. No deployed/live ESI +claim is made here. The narrow `beginSync` expected-authority fence is part of T06's +atomic bridge contract: it prevents transport cache authority from being published +under a different T05 grant/revision/credential fence; it is not T07 durable worker +orchestration. ## Execution model @@ -138,13 +141,15 @@ Implement the prototype identity enforcement needed by the pipeline: revalidate ### T07 — Durable asset queue and worker -**Own:** proposed `workers/sync.ts`, feature-local job/credential modules, job schema/store operations, `/api/sync` and replacement of request-bound sync orchestration; Compose wiring follows T01. +**Status:** implemented local prototype; no deployment, live ESI, or production-readiness claim. + +**Own:** `workers/sync.ts`, feature-local job/credential modules, job schema/store operations, `/api/sync` and replacement of request-bound sync orchestration; Compose wiring and restricted operator startup. **Deliver:** enqueue/status instead of long-running request work; typed deduplicated jobs, bounded attempts/due times, cancellation, fair scheduling/backpressure, shared cooldown/error budgets, renewable leases and fenced success **and failure** publication. Stage/validate assets before active-generation publication and retain validated `isSingleton` facts throughout ingestion. -Share a per-character refresh lease across web and workers, retaining token CAS. At-least-once delivery is not exactly-once remote refresh: an uncertain rotated result produces actionable reauthorization instead of automatic replay. Keep reference extraction separate and minimize worker secrets. Public feeds must be coordinated rather than independently refreshed per process. +Share a per-character refresh lease across web and workers, retaining token CAS. At-least-once delivery is not exactly-once remote refresh: an uncertain rotated result produces actionable reauthorization instead of automatic replay. Keep reference extraction separate and minimize worker secrets. Public feeds must be coordinated rather than independently refreshed per process. The worker requires the distinct `CONVEX_SYNC_WORKER_SECRET`; standalone startup uses a worker-only environment and Compose startup uses the explicit `worker` profile. -**Done:** process kill/restart, long paging beyond the old five-minute lease, duplicate delivery, cancellation, scope loss, stale-worker completion/failure, refresh races and uncertain rotation preserve the newest valid generation. Caps fail visibly. User-supplied jobs cannot gain system authority. Isolated migration/rollback and minimal worker configuration are exercised. +**Verified locally:** process kill/restart, long paging beyond the old five-minute lease, duplicate delivery, cancellation, scope loss, stale-worker completion/failure, refresh races, uncertain rotation, restricted runtime startup and configuration boundaries. The full suite currently records 204 passing tests. T08 enrichment integration at the post-assets/pre-publication seam remains pending; production migration/release evidence remains deferred. ### T08 — Private locations, custom names and containment diff --git a/docs/restore-rehearsal.md b/docs/restore-rehearsal.md index 686a83b..a5bddc3 100644 --- a/docs/restore-rehearsal.md +++ b/docs/restore-rehearsal.md @@ -49,7 +49,7 @@ chmod 700 "$work" # Do not let an ambient shell, repository .env, or prior Compose project alter # this rehearsal. The explicit env file below contains only non-secret values; # synthetic key values are loaded separately from the mode-0600 key file. -unset COMPOSE_FILE COMPOSE_ENV_FILES COMPOSE_PROJECT_NAME COMPOSE_PROFILES DOCKER_HOST APP_MODE APP_ORIGIN ORIGIN APP_RELEASE_ID APP_CAPABILITY_ALLOWLIST CONVEX_URL PUBLIC_CONVEX_URL CONVEX_SITE_ORIGIN CONVEX_PORT CONVEX_SITE_PORT WEB_PORT CONVEX_IMAGE CONVEX_DASHBOARD_IMAGE EVE_CALLBACK_URL EVE_CLIENT_ID EVE_CLIENT_SECRET ESI_USER_AGENT TYPESAFE_API_KEY APP_AUTH_ISSUER TOKEN_ENCRYPTION_KEY CONVEX_BRIDGE_SECRET APP_AUTH_PRIVATE_KEY APP_AUTH_JWKS CONVEX_SELF_HOSTED_URL CONVEX_SELF_HOSTED_ADMIN_KEY +unset COMPOSE_FILE COMPOSE_ENV_FILES COMPOSE_PROJECT_NAME COMPOSE_PROFILES DOCKER_HOST APP_MODE APP_ORIGIN ORIGIN APP_RELEASE_ID APP_CAPABILITY_ALLOWLIST CONVEX_URL PUBLIC_CONVEX_URL CONVEX_SITE_ORIGIN CONVEX_PORT CONVEX_SITE_PORT WEB_PORT CONVEX_IMAGE CONVEX_DASHBOARD_IMAGE EVE_CALLBACK_URL EVE_CLIENT_ID EVE_CLIENT_SECRET ESI_USER_AGENT TYPESAFE_API_KEY APP_AUTH_ISSUER TOKEN_ENCRYPTION_KEY CONVEX_BRIDGE_SECRET CONVEX_SYNC_WORKER_SECRET APP_AUTH_PRIVATE_KEY APP_AUTH_JWKS CONVEX_SELF_HOSTED_URL CONVEX_SELF_HOSTED_ADMIN_KEY export CONVEX_BIND_ADDRESS=127.0.0.1 CONVEX_PORT=$((3200 + RANDOM % 500)) CONVEX_SITE_PORT=$((3700 + RANDOM % 500)) export WEB_BIND_ADDRESS=127.0.0.1 WEB_PORT=$((4200 + RANDOM % 500)) export APP_MODE=live APP_RELEASE_ID=restore-fixture APP_CAPABILITY_ALLOWLIST=asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions diff --git a/docs/roadmap.md b/docs/roadmap.md index 45449b0..1c4b11d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,7 +1,7 @@ # Implementation roadmap **Status:** planning proposal, not implementation or a public-launch declaration. -**Baseline:** the operator has verified real EVE sign-in, asset synchronization, and matching Assets/chat reference valuations. The earlier implementation gate recorded 97 passing tests; local prototype checks are rerun as features land. +**Baseline:** prior checkpoints historically recorded live sign-in, asset synchronization and matching Assets/chat reference valuations for one character (see [verification](verification.md)); that history is not new T07 evidence. This T07 checkpoint records a local, disposable prototype only, with no new deployed/live ESI or production-readiness claim. T07 durable synchronization is implemented and verified locally with the current 204-test suite. T08 location/containment enrichment remains a separate feature whose post-assets/pre-publication integration is still pending. **Evidence:** [architecture](architecture.md), [milestone 1](milestone.md), [verification](verification.md), [ESI research](research/eve.md), [expansion data/permission matrix](research/expansion-data-contracts.md), and [reference operations](reference-operations.md). ## 1. Decisions confirmed for this roadmap @@ -16,6 +16,10 @@ New service/table/path names below are proposed implementation boundaries, not existing APIs. Later phases need focused design and current contract verification before coding; they are not one large implementation task. +### T07 prototype checkpoint + +T07 is implemented as a local prototype, not a deployment claim. It requires the distinct `CONVEX_SYNC_WORKER_SECRET` in Convex and the worker environment, alongside `TOKEN_ENCRYPTION_KEY`, EVE client credentials and `ESI_USER_AGENT`. Start Compose workers only through the explicit `worker` profile; standalone development uses a worker-only `.env.worker`. T08 core modules and their integration at the post-assets/pre-publication seam remain pending. + ## 2. What exists versus what still needs building | Area | Current implementation | Important gap | @@ -24,7 +28,7 @@ New service/table/path names below are proposed implementation boundaries, not e | Assets | Complete staged snapshots, tenant gates, exact reference totals | Real search/filtering, paging, containment, custom names, meaningful locations, history | | References | 53,001 types; recipes, dogma and system/gate tables; immutable publication | Runtime reader exposes only types/version; station table currently empty; search and domain-specific readers missing | | Prices | Global ESI average prices, held by the web process | Durable public feed coordination, regional/private books, history, depth allocation, fees | -| Sync | On-demand work inside web requests; generation publication and token CAS | `jobs` is schema-only; no durable dispatcher, crash recovery, shared refresh lease or global worker budgets | +| Sync | T07 durable queued jobs, dedicated worker, leases, fenced generation publication, cancellation/recovery and shared refresh coordination | T08 enrichment integration and production release evidence remain pending | | Finances | Signed integer-cent money helpers | Wallets, journals, transactions, own orders, history/coverage and reconciliation | | Industry | Recipe data plus placeholder interfaces/economics fields | Owned blueprint instances, skills/jobs, facility assumptions, formulas and optimizer | | Navigation | Exact light-year distance helper and universe data | Gate-distance service, ship-aware capital graph, dynamic constraints and qualified access evidence | diff --git a/docs/security.md b/docs/security.md index 3e81617..92f8337 100644 --- a/docs/security.md +++ b/docs/security.md @@ -7,10 +7,10 @@ These are requirements for the first implementation, not a claim of a production - Connecting another character is an authenticated, explicit operation. Callback must preserve the initiating user/session binding. Character ownership conflicts fail closed; there is no implicit merge or transfer. - Refresh tokens are envelope encrypted using a server-held encryption key with authenticated associated data. No raw refresh token or client secret may be a browser property, public Convex query field, URL parameter, log or exception message. - Browser sessions use high-entropy opaque identifiers, server-side hashes, HttpOnly/SameSite cookies and bounded expiration. Application JWTs use a dedicated issuer/audience/key and short expiry, not EVE bearer tokens. Queries independently enforce the application identity and ownership. -- Server bridge authority is distinct from the deployment administrator credential and browser sessions. All privileged Convex entrypoints require it. Deployment keys are not application runtime defaults. +- Server bridge authority is distinct from the deployment administrator credential and browser sessions. Web/session operations require the general bridge secret; worker lifecycle operations require a separate `CONVEX_SYNC_WORKER_SECRET` and expose no login, session, tenant-administration or enqueue/cancel authority. Deployment keys are not application runtime defaults. - State-changing browser requests require an exact allowed origin. Normal app pages use `Referrer-Policy: same-origin`: global `no-referrer` would make native POST navigations send `Origin: null` and fail CSRF verification. Auth responses retain `no-referrer` so callback grant URLs never become referrers. Null/missing/foreign origins remain rejected. No arbitrary user-provided fetch URLs, SQL identifiers, executable function names or tool invocations. - Market/ESI errors are redacted before display. Observability reports route, status, timings and cache/rate-limit signals, never Authorization headers, OAuth codes or secrets. -- Authenticated ESI caches are keyed by character and authorization context. A partial sync cannot replace a complete snapshot. Token refresh is serialized for a character; stale token rotations cannot overwrite newer credentials. +- Authenticated ESI caches are keyed by character and authorization context. A partial sync cannot replace a complete snapshot. Durable jobs use tokenized renewable leases and fenced success/failure publication; duplicate staging is idempotent only for identical payloads. Token refresh is serialized per character with durable CAS; stale token rotations cannot overwrite newer credentials, and uncertain rotation outcomes fail closed as `reconnect_required` rather than replaying. - Fixture/demo data is explicitly opt-in and visibly marked. Live errors never trigger a silent fallback to demo assets or prices; live reference readers require a production-profile artifact. - Reference archives are untrusted input. The production importer reads members without extracting arbitrary filesystem paths, rejects unsafe paths and unsupported member types, and enforces compressed/decompressed/member limits, XZ memory limits, archive checksums, and schema validation. Artifact paths/manifests must remain inside the configured reference directory. Cache reuse and fallback verify canonical hashes and the active artifact’s bound provenance; fixture caches cannot satisfy a production import. - Infisical bootstrap credentials stay in an owner-only local file, separate from application keys. The launcher pins the approved instance and selected project/environment/path, suppresses raw credential-command output, and does not pass bootstrap credentials or the vault token to the application. Application secrets are injected only from an allowlist; deployment admin access requires an explicit operator command. Docker/host administrators can still inspect container environment values. diff --git a/docs/verification.md b/docs/verification.md index 57ed780..ded3640 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -1,6 +1,6 @@ # Verification record -Initial milestone verified locally on 2026-09-18; live development rollout checked on 2026-09-19 UTC with Bun 1.3.14. Dependency versions are pinned in `package.json`/`bun.lock`. +This record retains historical verification facts from earlier checkpoints. The current T07 checkpoint is a local/disposable prototype only: it makes no new deployed, live-ESI, or production-readiness claim. Dependency versions are pinned in `package.json`/`bun.lock`. ## Initial offline/disposable-backend milestone @@ -8,19 +8,91 @@ Initial milestone verified locally on 2026-09-18; live development rollout check - `bun test`: offline tests exercise domain ownership/freshness, precise integer-cent accumulation, unknown/unpriced/BPC behavior, typed JEV routing and abstention, cryptographic envelopes, PKCE, SSO JWT checks, Convex ownership/session/snapshot rules, ESI cache/pagination/limiter behavior, and reference promotion/rollback/source failure handling. Final run: **66 passed, 0 failed**, across 10 files. - `bun run build`: production Bun server builds. `svelte-adapter-bun` may emit an unresolved `bun:sqlite` warning while leaving that Bun builtin external. This is not a missing npm dependency: the resulting server and container were executed successfully under Bun. - `scripts/smoke-web.ts`: executed against both the built production server and the final Docker web container in explicit fixture mode. Assistant, Characters and Assets SSR routes succeeded. `/value` and `/assets` returned the same 208,600,000 integer cents (2,086,000.00 ISK), two characters and two excluded/unpriced stacks. Foreign origins and oversized request bodies were rejected. -- Browser-bundle inspection: no references to `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `APP_AUTH_PRIVATE_KEY`, `refresh_token` or `TYPESAFE_API_KEY` in `build/client`. +- Browser-bundle inspection: no references to `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `CONVEX_SYNC_WORKER_SECRET`, `APP_AUTH_PRIVATE_KEY`, `refresh_token` or `TYPESAFE_API_KEY` in `build/client`. - `docker compose config --quiet` and `docker build`: passed. The containerized fixture worker also executed with `--network none`; no archives were fetched. -- Real self-hosted Convex: deployed functions/schema/auth to an isolated disposable backend and executed `scripts/smoke-convex.ts`. Verified session lookup, asset publication, preservation after failed staging, cross-user isolation, safe status projection, correctly signed wrong-audience JWT rejection, and session revocation. This caught and fixed an incorrect attempt to read reserved `aud` as a Convex identity custom claim; audience enforcement belongs in the backend `applicationID` auth configuration. +No live EVE credentials, production volumes, or deployed endpoints were used for this T07 checkpoint. Optional disposable Convex smoke procedures remain documented below but are not evidence of a deployment. -The disposable backend/web containers were stopped and test-only environment files and keys removed after verification. Typecheck and all 66 offline tests were rerun successfully without those files. +## Historical verification from prior checkpoints -Tested backend image: +The following evidence was recorded before T07 and is retained as historical context. It is not evidence that this new T07 worker prototype has been deployed or exercised against live ESI. + +- The initial disposable Convex smoke deployed functions/schema/auth to an isolated backend and verified session lookup, asset publication, preservation after failed staging, cross-user isolation, safe status projection, correctly signed wrong-audience JWT rejection, and session revocation. It caught and fixed an incorrect attempt to read reserved `aud` as a Convex identity custom claim; audience enforcement belongs in the backend `applicationID` auth configuration. +- The disposable backend/web containers were stopped and test-only environment files and keys removed after that verification. Typecheck and all 66 offline tests were rerun successfully without those files. +- Historical tested backend image: + + ```text + ghcr.io/get-convex/convex-backend@sha256:3aa2ccd6697a75de53948b1ee531d7c9688fe09ef360748d75df46dfc888f69a + ``` + +### Historical live development rollout — 2026-09-19 UTC + +- The integrated gate recorded **97 tests passed, 0 failed, across 16 files**; Svelte/TypeScript check passed with 0 errors/0 warnings, and the Docker web build passed. +- An operator-created Infisical project was connected through a read-only Universal Auth identity. Generated app encryption/signing/bridge keys were imported, compared against the staging file, and locally verified for encryption round-trip and signing/JWKS consistency; the temporary `.env.keys` was removed and secret values were not displayed. +- Persistent self-hosted Convex schema/functions/auth configuration was deployed. Its administrator credential was derived locally for explicit deployment commands, not stored in the web environment. +- Pangolin/UniFi routing was verified: `https://dev-convex.evebot.app/version` returned HTTP 200 and a real WSS upgrade with `Origin: https://dev.evebot.app` passed. Pangolin authentication remained allowed on the main app but was disabled on the Convex API resource by the operator. +- One real TypeSafe request through `JevIntentRouter` returned `asset_valuation`, model `jev-1.13.0`, confidence 1, in 915 ms; this proves connectivity/routing for that sample, not calibration. +- One real public ESI `/markets/prices` request passed using pinned compatibility `2026-08-18`, returning fresh integer-cent estimates for Tritanium and Rifter. These remain global reference estimates, not Jita bids or executable prices. +- Real Reference Data/Hoboleaks/CCP archives totaling **114,291,615 compressed bytes (~109 MiB)** were parsed. The corrected Docker worker compiled them with `--network none`, a 4 GiB memory limit and two CPUs, explicitly targeting `/app/data/reference` in the shared volume. Peak RSS was not measured. The full online download path has fixture-based receipt/validator tests; the real full rebuild used already downloaded archives rather than fetching another copy. +- The web container opened the deployed production artifact, resolved Tritanium/Rifter, matched manifest/table counts, and passed SQLite integrity/foreign-key checks: **53,001 types, 5,082 blueprints, 8,490 systems, 13,978 gates, 21 explicitly recorded unresolved blueprint references**. Deployed version: `b9afed7830756a5e617fec31a2a88c917f07b8f64d2bc719e2176fb628c1e98b`. +- Live anonymous web smoke passed after deployment: SSR routes, no demo fallback, asset/chat authentication gates, session-token protection, and CSRF rejection. A scan of **13 deployed client artifacts plus anonymous SSR output** found no managed secret values; bootstrap tokens and the deployment-admin key were absent from the web environment. + +A subsequent operator test exposed `ERR_INVALID_RESPONSE` when submitting Connect Character: the global `no-referrer` response policy made native browser form submissions send `Origin: null`, which SvelteKit rejected before OAuth initiation. An isolated temporary-profile Firefox test reproduced that behavior and confirmed that `same-origin` preserves the native form Origin. App pages now use `same-origin` while auth responses retain `no-referrer`; null/missing/foreign origins remain rejected. The rebuilt deployment passed header and anonymous endpoint checks. EVE discovery and a server-to-Convex read also passed; diagnostics wrote no OAuth/character records. The operator then reached EVE login but reported a generic callback failure. Redacted callback diagnostics identified `scopes_invalid` after token verification; the parser was updated to accept EVE's singleton scope-string representation as well as arrays, with signed-token login/refresh regressions and malformed-claim rejection. The revised image and anonymous smoke passed. The operator subsequently confirmed successful character authorization and asset synchronization, including refresh-grant verification and owner continuity before asset publication, and confirmed that live Assets and chat `/value` returned matching totals without sharing the private amount. + +Historical audit regressions covered native-form/CSRF boundaries, shared artifact-directory precedence, live readers rejecting fixture profiles, offline no-network behavior, archive receipts/replay, ZIP/TAR checksums and limits, duplicate records/JSON keys, Hoboleaks metadata, and immutable cache provenance/rollback/reuse. See [live setup](live-setup.md) and [reference operations](reference-operations.md). + +## T07 durable worker boundary regressions + +- Full local `bun test`: **204 passed, 0 failed** across 27 files. +- `bun test tests/t07-sync.test.ts tests/t07-worker-runtime.test.ts tests/t07-route-boundaries.test.ts`: deterministic queue/fencing/retry regressions plus runtime alias/startup, missing-job/AbortSignal polling, HTTP 429/503 structured outcomes, forbidden worker-environment, hostile-origin, session-ordering and bounded-body checks (31 passed in the focused run). +- `bun build workers/sync.ts --target bun`: produces the same alias-resolved worker artifact copied by the Docker build to `build/sync-worker.js`. A no-cache Docker image build completed, and `docker run --rm --network none` started the image's bundled worker through `scripts/run-sync-worker.ts` with inert Convex/EVE values and `SYNC_WORKER_STARTUP_CHECK=1`; no live keys or network calls were used. `scripts/run-sync-worker.ts` is separately tested to reject general bridge/admin/JWT/JEV credentials without echoing them. +- The operational startup is `bun run secrets:run --worker -- docker compose --profile worker up -d --build sync-worker`; `--deployment`/`--configure` is reserved for Convex administration and configuration, not worker startup. Plain `docker compose up` leaves the worker stopped. + +### T07 prototype stage manifest + +The T07 stage includes the following files. T08 core files are intentionally excluded: ```text -ghcr.io/get-convex/convex-backend@sha256:3aa2ccd6697a75de53948b1ee531d7c9688fe09ef360748d75df46dfc888f69a +.env.example +Dockerfile +README.md +compose.yaml +docs/architecture.md +docs/design-records/t07-durable-sync.md +docs/live-setup.md +docs/parallel-worktrees.md +docs/restore-rehearsal.md +docs/roadmap.md +docs/security.md +docs/verification.md +package.json +scripts/configure-convex.ts +scripts/generate-keys.ts +scripts/lib/infisical.ts +scripts/release-check.ts +scripts/run-sync-worker.ts +scripts/secrets.ts +src/convex/functions.ts +src/convex/schema.ts +src/lib/client/sync-queue.ts +src/lib/domain/model.ts +src/lib/server/application.ts +src/lib/server/store.ts +src/routes/+page.svelte +src/routes/api/sync/+server.ts +src/routes/api/sync/cancel/+server.ts +src/routes/api/sync/status/+server.ts +src/routes/assets/+page.svelte +tests/infisical.test.ts +tests/t07-route-boundaries.test.ts +tests/t07-sync.test.ts +tests/t07-worker-runtime.test.ts +tests/workflow-config.test.ts +workers/sync.ts ``` -## Reproducing the real Convex smoke +Excluded T08 core files: `src/lib/domain/inventory-location.ts`, `src/lib/server/inventory/`, `tests/fixtures/inventory/`, `tests/inventory-containment.test.ts`, and `tests/inventory-location.test.ts`. + +## Optional disposable Convex smoke (not part of this checkpoint) Use a **disposable** backend, set its `CONVEX_SELF_HOSTED_URL` and admin key, generate application keys and configure/deploy as in the README, then run: @@ -30,25 +102,13 @@ SMOKE_ALLOW_WRITE=1 bun scripts/smoke-convex.ts This writes synthetic users/characters and must not be casually pointed at a production deployment. It needs `CONVEX_BRIDGE_SECRET`, `APP_AUTH_ISSUER` and `APP_AUTH_PRIVATE_KEY` in the calling environment. A wrong-audience test is signed locally; no EVE identity is impersonated and no real EVE account is used. -## Live development rollout — 2026-09-19 UTC +## Deferred deployment and live-provider evidence -- Latest integrated gate (including native-form, callback diagnostics, and singleton-scope compatibility): **97 tests passed, 0 failed, across 16 files**; Svelte/TypeScript check passed with 0 errors/0 warnings. Docker web build passed. -- Connected the operator-created Infisical project through a read-only Universal Auth identity. Generated app encryption/signing/bridge keys were imported by the operator, compared against the staging file, and locally verified for encryption round-trip and signing/JWKS consistency. The temporary `.env.keys` was then removed. Secret values were not displayed. -- Persistent self-hosted Convex schema/functions/auth configuration deployed. Its administrator credential is derived locally for explicit deployment commands, not stored in the web environment. -- Pangolin/UniFi routing was verified: `https://dev-convex.evebot.app/version` returned HTTP 200 and a real WSS upgrade with `Origin: https://dev.evebot.app` passed. Pangolin authentication remains allowed on the main app but was disabled on the Convex API resource by the operator. -- One real TypeSafe request through `JevIntentRouter` returned `asset_valuation`, model `jev-1.13.0`, confidence 1, in 915 ms. This proves connectivity/routing for that sample, not calibration. -- One real public ESI `/markets/prices` request passed using pinned compatibility `2026-08-18`, returning fresh integer-cent estimates for Tritanium and Rifter. These remain global reference estimates, not Jita bids or executable prices. -- Real Reference Data/Hoboleaks/CCP archives totaling **114,291,615 compressed bytes (~109 MiB)** were parsed. The corrected Docker worker compiled them with `--network none`, a 4 GiB memory limit and two CPUs, explicitly targeting `/app/data/reference` in the shared volume. Peak RSS was not measured. The full online download path has fixture-based receipt/validator tests; the real full rebuild used the already downloaded archives rather than fetching another copy. -- The web container's own read-only repository successfully opened the deployed production artifact, resolved Tritanium/Rifter, matched manifest/table counts, and passed SQLite integrity/foreign-key checks: **53,001 types, 5,082 blueprints, 8,490 systems, 13,978 gates, 21 explicitly recorded unresolved blueprint references**. Deployed version: `b9afed7830756a5e617fec31a2a88c917f07b8f64d2bc719e2176fb628c1e98b`. -- Live anonymous web smoke passed after deployment: SSR routes, no demo fallback, asset/chat authentication gates, session-token protection, and CSRF rejection. A scan of **13 deployed client artifacts plus anonymous SSR output** found no managed secret values; bootstrap tokens and the deployment-admin key were absent from the web environment. - -A subsequent operator test exposed `ERR_INVALID_RESPONSE` when submitting Connect Character. The global `no-referrer` response policy made native browser form submissions send `Origin: null`; SvelteKit rejected them before OAuth initiation. An isolated, temporary-profile Firefox test reproduced that behavior and confirmed that `same-origin` preserves the native form Origin. App pages now use `same-origin` while auth responses retain `no-referrer`; null/missing/foreign origins remain rejected. The rebuilt deployment passed header and anonymous endpoint checks. EVE discovery and a server-to-Convex read also passed; diagnostics wrote no OAuth/character records. The operator subsequently reached EVE login but reported a generic failure after callback. Redacted callback-stage diagnostics were added and deployed: only fixed failure categories and allowlisted protocol status/claim names are retained; provider descriptions, JWT payloads, credentials, codes, cookies, and user identifiers are excluded. The next redacted callback diagnostic identified `scopes_invalid` after token verification. The parser now accepts EVE's singleton scope-string representation as well as arrays, with signed-token login/refresh regressions and malformed-claim rejection. The revised image and anonymous smoke passed. After that deployment, the operator confirmed successful character authorization and asset synchronization. Explicit sync exercises refresh-grant verification and owner continuity before asset publication. The operator also confirmed that the live Assets page and chat `/value` return matching totals, without sharing the private amount. This completes the initial live sign-in/sync/reference-valuation verification for one character. - -Audit regressions cover native-form header/CSRF boundaries, shared artifact-directory precedence, live readers rejecting fixture profiles, offline no-network behavior, archive receipts/replay, ZIP CRC/length/duplicate records, TAR checksums/end markers, duplicate JSON keys, Hoboleaks file metadata, and immutable cache provenance/rollback/reuse. See [live setup](live-setup.md) and [reference operations](reference-operations.md). +No deployed endpoint, live ESI request, live EVE credential, or production rollout is claimed for this T07 checkpoint. Existing operational procedures remain reference material only; deployment, migration/rollback, live-provider, browser failure-injection and production-readiness evidence are deferred. See [live setup](live-setup.md) and [reference operations](reference-operations.md). ## Not claimed as verified -- Live revocation/character-transfer scenarios and additional-character linking; successful sign-in/sync for one real character is now operator-confirmed. +- Live revocation/character-transfer scenarios, additional-character linking, live ESI behavior and deployed-artifact evidence are not claimed for this prototype. - Broad TypeSafe inference quality or the proposed 0.80 confidence threshold's calibration on EVE requests. - Compatibility with arbitrary future archive releases, station-data coverage, large-scale memory/performance benchmarks, or production-size market analytics. - End-to-end browser automation/visual accessibility audit; the web smoke verifies the built server and endpoints, not all browser interactions. diff --git a/package.json b/package.json index 8cf8cd6..4594e0a 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "reference:fixture": "bun workers/reference.ts --fixture", "reference:production": "bun workers/reference.ts --production", "reference:rollback": "bun workers/reference.ts --rollback", + "sync:worker": "bun --no-env-file scripts/run-sync-worker.ts", "convex:dev": "bunx convex dev", "convex:configure": "bun scripts/configure-convex.ts", "keys:generate": "bun scripts/generate-keys.ts", diff --git a/scripts/configure-convex.ts b/scripts/configure-convex.ts index 40bf7bf..1fd720d 100644 --- a/scripts/configure-convex.ts +++ b/scripts/configure-convex.ts @@ -1,7 +1,8 @@ export {}; // Run only against your intended self-hosted deployment. Values are never printed. -const requiredNames = ['CONVEX_BRIDGE_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const; +const requiredNames = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const; if (!process.env.CONVEX_SELF_HOSTED_URL || !process.env.CONVEX_SELF_HOSTED_ADMIN_KEY) throw new Error('Set self-hosted URL and admin key before configuring Convex.'); +if (process.env.CONVEX_SYNC_WORKER_SECRET && process.env.CONVEX_SYNC_WORKER_SECRET === process.env.CONVEX_BRIDGE_SECRET) throw new Error('Worker and general bridge secrets must be distinct.'); for (const name of requiredNames) { const value = process.env[name]; if (value === undefined || (name !== 'APP_CAPABILITY_ALLOWLIST' && !value)) { diff --git a/scripts/generate-keys.ts b/scripts/generate-keys.ts index 1eb62cd..9ab408d 100644 --- a/scripts/generate-keys.ts +++ b/scripts/generate-keys.ts @@ -6,8 +6,9 @@ const publicJwk = { ...await exportJWK(publicKey), kid: 'app-v1', alg: 'RS256', const values = { TOKEN_ENCRYPTION_KEY: randomBytes(32).toString('base64'), CONVEX_BRIDGE_SECRET: randomBytes(32).toString('base64url'), + CONVEX_SYNC_WORKER_SECRET: randomBytes(32).toString('base64url'), APP_AUTH_PRIVATE_KEY: Buffer.from(await exportPKCS8(privateKey)).toString('base64'), APP_AUTH_JWKS: `data:application/json;base64,${Buffer.from(JSON.stringify({ keys: [publicJwk] })).toString('base64')}` }; await writeFile('.env.keys', Object.entries(values).map(([k, v]) => `${k}='${v}'`).join('\n') + '\n', { flag: 'wx', mode: 0o600 }); -console.log('Wrote .env.keys (0600). Import its four values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.'); +console.log('Wrote .env.keys (0600). Import its five values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.'); diff --git a/scripts/lib/infisical.ts b/scripts/lib/infisical.ts index 4598b77..66736d0 100644 --- a/scripts/lib/infisical.ts +++ b/scripts/lib/infisical.ts @@ -12,9 +12,11 @@ export interface VaultProject { export const BOOTSTRAP_KEYS = ['INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET'] as const; export const APPLICATION_KEYS = [ 'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY', 'TOKEN_ENCRYPTION_KEY', - 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', + 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', ] as const; -export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const; +export const WORKER_APPLICATION_KEYS = ['EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_SYNC_WORKER_SECRET'] as const; +export const CONFIGURE_APPLICATION_KEYS = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_JWKS'] as const; +export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const; export const ADMIN_KEY = 'CONVEX_SELF_HOSTED_ADMIN_KEY'; export type Environment = Record; export type CliRunner = (args: string[], env: Environment) => Promise; @@ -157,7 +159,7 @@ const PUBLIC_KEYS = new Set([ 'WEB_BIND_ADDRESS', 'CONVEX_BIND_ADDRESS', 'WEB_PORT', 'CONVEX_PORT', 'CONVEX_SITE_PORT', 'CONVEX_IMAGE', 'CONVEX_DASHBOARD_IMAGE', 'RELEASE_ENV', ]); -export function applicationEnvironment(base: Environment, secrets: Record, publicConfig: Environment, deployment = false): Environment { +export function applicationEnvironment(base: Environment, secrets: Record, publicConfig: Environment, deployment = false, target: 'web' | 'worker' | 'configure' = 'web'): Environment { const env = cleanEnvironment(base); // Public file is the configured deployment, not defaults inherited from Bun's .env.local. for (const [key, value] of Object.entries(publicConfig)) { @@ -165,10 +167,9 @@ export function applicationEnvironment(base: Environment, secrets: Record key !== 'CONVEX_SYNC_WORKER_SECRET'); + for (const key of allowedKeys) if (secrets[key] !== undefined) env[key] = secrets[key]; if (deployment && secrets[ADMIN_KEY]) env[ADMIN_KEY] = secrets[ADMIN_KEY]; return env; } diff --git a/scripts/release-check.ts b/scripts/release-check.ts index d560b36..c51bc89 100644 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -65,7 +65,7 @@ function trackedFiles(): string[] { return result.stdout.split('\n').filter(Boolean); } -const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token']; +const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token']; function browserBundleFiles(directory = 'build/client'): string[] { try { statSync(directory); } catch { return []; } const files: string[] = []; diff --git a/scripts/run-sync-worker.ts b/scripts/run-sync-worker.ts new file mode 100644 index 0000000..04594ba --- /dev/null +++ b/scripts/run-sync-worker.ts @@ -0,0 +1,42 @@ +import { existsSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { join } from 'node:path'; + +const forbidden = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY']; +const workerEnvironmentKeys = [ + 'PATH', 'HOME', 'TMPDIR', 'NODE_ENV', 'LANG', 'LC_ALL', 'TZ', 'NODE_EXTRA_CA_CERTS', + 'CONVEX_URL', 'CONVEX_SYNC_WORKER_SECRET', 'TOKEN_ENCRYPTION_KEY', + 'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'EVE_CALLBACK_URL', 'ESI_USER_AGENT', + 'SYNC_WORKER_ID', 'SYNC_WORKER_LEASE_MS', 'SYNC_WORKER_HEARTBEAT_MS', 'SYNC_WORKER_POLL_MS', 'SYNC_WORKER_STARTUP_CHECK', +] as const; +export function workerEnvironment(base: Record): Record { + return Object.fromEntries(workerEnvironmentKeys.flatMap((key) => base[key] === undefined ? [] : [[key, base[key] as string]])); +} + +export async function runSyncWorkerLauncher(): Promise { +for (const name of forbidden) { + if (process.env[name]) throw new Error(`Sync worker refuses forbidden runtime credential: ${name}`); +} +if (!process.env.CONVEX_SYNC_WORKER_SECRET) throw new Error('CONVEX_SYNC_WORKER_SECRET is required for the sync worker'); + +const bundled = join(process.cwd(), 'build', 'sync-worker.js'); +let entry = bundled; +let temporary = false; +if (!existsSync(entry)) { + entry = join('/tmp', `eve-sync-worker-${process.pid}.js`); + const result = await Bun.build({ entrypoints: ['workers/sync.ts'], target: 'bun', outdir: '/tmp', naming: `eve-sync-worker-${process.pid}.js`, minify: false }); + if (!result.success) throw new Error('Could not bundle the sync worker'); + temporary = true; +} +const child = Bun.spawn(['bun', entry], { env: workerEnvironment(process.env), stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' }); +const interrupt = () => child.kill('SIGINT'); +const terminate = () => child.kill('SIGTERM'); +process.on('SIGINT', interrupt); process.on('SIGTERM', terminate); +try { process.exitCode = await child.exited; } +finally { + process.off('SIGINT', interrupt); process.off('SIGTERM', terminate); + if (temporary) await rm(entry, { force: true }); +} +} + +if (import.meta.main) await runSyncWorkerLauncher(); diff --git a/scripts/secrets.ts b/scripts/secrets.ts index c9e9e99..a43fab1 100644 --- a/scripts/secrets.ts +++ b/scripts/secrets.ts @@ -10,7 +10,7 @@ const HELP = `Infisical-backed development deployment (secret values are never p bun run secrets:run -- [arguments...] bun run secrets:run --deployment -- [arguments...] -Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env) +Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env), --worker to inject only worker-safe application secrets, or --deployment --configure for Convex function configuration. Project metadata: .infisical.json. Public deployment configuration: config/dev.env. Only allowlisted app secrets are injected. --deployment additionally permits the Convex admin key and, when absent from the vault, derives it from the local persistent backend without printing it. @@ -23,22 +23,29 @@ export function parseArguments(args: string[]) { let command: string[] = []; let credentialsFile = join(homedir(), '.config/evebot/infisical.env'); let deployment = false; + let worker = false; + let configure = false; for (let i = 0; i < rest.length; i++) { if (rest[i] === '--') { command = rest.slice(i + 1); break; } if (rest[i] === '--credentials-file' && rest[i + 1] && !rest[i + 1].startsWith('--')) credentialsFile = resolve(rest[++i]); else if (rest[i] === '--deployment') deployment = true; + else if (rest[i] === '--worker') worker = true; + else if (rest[i] === '--configure') configure = true; // `bun run` consumes its own -- separator before forwarding script arguments. else if (action === 'run' && !rest[i].startsWith('-')) { command = rest.slice(i); break; } else throw new Error('Unknown or incomplete secrets helper option'); } if (action === 'run' && !command.length) throw new Error('Supply a trusted command after --'); if (action === 'status' && command.length) throw new Error('status does not execute a command'); - return { action, command, credentialsFile, deployment }; + if (worker && deployment) throw new Error('Worker secret mode cannot request deployment admin credentials'); + if (worker && configure) throw new Error('Worker and configure secret modes are mutually exclusive'); + if (configure && !deployment) throw new Error('--configure requires --deployment so the Convex admin credential is available'); + return { action, command, credentialsFile, deployment, worker, configure }; } async function main(args: string[]): Promise { if (!args.length || args[0] === '--help') { console.log(HELP); return; } - const { action, command, credentialsFile, deployment } = parseArguments(args); + const { action, command, credentialsFile, deployment, worker, configure } = parseArguments(args); const project = validateProject(JSON.parse(await readFile(new URL('../.infisical.json', import.meta.url), 'utf8'))); const credentials = await readCredentials(credentialsFile); const secrets = await fetchSecrets(project, credentials); @@ -54,7 +61,7 @@ async function main(args: string[]): Promise { return; } const publicConfig = parseEnv(await readFile(new URL('../config/dev.env', import.meta.url), 'utf8')); - const env = applicationEnvironment(process.env, secrets, publicConfig, deployment); + const env = applicationEnvironment(process.env, secrets, publicConfig, deployment, worker ? 'worker' : configure ? 'configure' : 'web'); if (deployment && !env[ADMIN_KEY]) env[ADMIN_KEY] = await localDeploymentAdmin(env); const child = Bun.spawn(command, { env, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' }); const interrupt = () => child.kill('SIGINT'); diff --git a/src/convex/functions.ts b/src/convex/functions.ts index 37599f6..c47bba9 100644 --- a/src/convex/functions.ts +++ b/src/convex/functions.ts @@ -20,6 +20,8 @@ const MAX_CHUNKS = 4_096; const MAX_ERROR_LENGTH = 160; const OAUTH_MAX_LIFETIME_MS = 10 * 60 * 1000; const SNAPSHOT_MAX_LIFETIME_MS = 60 * 60 * 1000; +const MAX_USER_ACTIVE_SYNC_JOBS = 8; +const SYNC_PROVIDER_GROUP = 'esi:assets'; const DATASET_SCOPE_REQUIREMENTS: Record = { assets: ['esi-assets.read_assets.v1'], wallet_balance: ['esi-wallet.read_character_wallet.v1'], @@ -33,6 +35,7 @@ const CONSENT_INTENTS = ['initial_login', 'link_character', 'upgrade_dataset', ' const CONSENT_CAPABILITIES = Object.keys(CAPABILITY_DEFINITIONS) as Capability[]; const bridgeArgs = { bridgeSecret: v.string() }; +const workerBridgeArgs = { workerSecret: v.string() }; const characterArgs = { userId: v.string(), characterId: v.number() }; const assetPageResultValidator = v.object({ @@ -57,6 +60,12 @@ function assertBridge(secret: string): void { } } +function assertSyncWorker(secret: string): void { + const expected = process.env.CONVEX_SYNC_WORKER_SECRET; + if (!expected || expected === process.env.CONVEX_BRIDGE_SECRET || !constantTimeEqual(secret, expected)) throw new Error("Sync worker authentication failed"); + assertDeploymentCapability('asset_sync'); +} + function assertDeploymentCapability(capability: 'asset_sync'): void { let enabled = false; try { @@ -81,6 +90,19 @@ function safeSyncError(_error: string): string { return "Asset synchronization failed"; } +function workerJob(job: any): boolean { + return job?.kind === 'asset_sync' && job?.dataset === 'assets' && job?.authority === 'user' && job?.ownerKind === 'user'; +} + +function authorityMatches(character: any, job: any): boolean { + return Boolean(character && character.authorizationStatus === 'active' + && character.scopes.includes('esi-assets.read_assets.v1') + && character.grantRevision === job.grantRevision + && character.authorizationEpoch === job.authorizationEpoch + && character.credentialVersion === job.credentialVersion + && scopesEqual(character.scopes, job.scopes ?? [])); +} + function newIdentifier(prefix: string): string { return `${prefix}_${crypto.randomUUID()}`; } @@ -109,6 +131,11 @@ function validActualScopes(scopes: readonly string[]): boolean { return new Set(scopes).size === scopes.length && scopes.every((scope) => SERVER_SCOPE_ALLOWLIST.includes(scope)); } +function scopesSubset(subset: readonly string[], superset: readonly string[]): boolean { + const allowed = new Set(superset); + return subset.every((scope) => allowed.has(scope)); +} + function datasetStatusesFor(character: any): any[] { return character.datasetStatuses; } @@ -138,7 +165,7 @@ function updateDatasetStatuses( invalidatedAt: timestamp, ...(dataset === 'assets' ? {} : {}), }); - if (dataset === 'assets' && (old?.status === 'ready' || old?.generation !== undefined || character.activeGeneration !== undefined)) assetsInvalidated = true; + if (dataset === 'assets') assetsInvalidated = true; continue; } // A newly authorized dataset has no coverage. Existing coverage remains @@ -458,7 +485,7 @@ export const bridgeCompleteLogin = mutationGeneric({ credentialVersion, authorizationEpoch, datasetStatuses: updated.statuses, - ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}), + ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}), }; if (character) { await ctx.db.patch(character._id, basePatch); @@ -492,6 +519,7 @@ export const bridgeCompleteLogin = mutationGeneric({ version: credentialVersion, }); } + if (character && (grantChanged || updated.assetsInvalidated || credentialVersion !== character.credentialVersion)) await invalidateCharacterJobs(ctx, resolvedUserId, args.characterId, timestamp); const sessionId = newIdentifier("session"); await ctx.db.insert("sessions", { @@ -546,7 +574,7 @@ export const bridgeSaveTokens = mutationGeneric({ assertBridge(args.bridgeSecret); if (!validActualScopes(args.scopes)) return false; const character = await findCharacter(ctx, args.userId, args.characterId); - if (!character || character.authorizationStatus !== "active") return false; + if (!character || character.authorizationStatus !== "active" || !scopesSubset(args.scopes, character.scopes ?? [])) return false; const record = await ctx.db .query("tokenRecords") .withIndex("by_user_character", (q: any) => @@ -568,8 +596,9 @@ export const bridgeSaveTokens = mutationGeneric({ grantRevision: nextRevision, credentialVersion: nextVersion, datasetStatuses: updated.statuses, - ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}), + ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}), }); + if (updated.assetsInvalidated || nextRevision !== character.grantRevision || nextVersion !== character.credentialVersion) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now()); if (args.returnAuthority) { return { saved: true, @@ -973,6 +1002,581 @@ export const bridgeFailSync = mutationGeneric({ }, }); +/** Return only fields safe for an authenticated status response. */ +function safeSyncJob(job: any): any { + const knownErrors = new Set(['Synchronization cancelled', 'Synchronization scheduled for retry', 'Authorization is no longer active', 'Asset synchronization failed']); + const error = typeof job.lastError === 'string' && knownErrors.has(job.lastError) ? job.lastError : undefined; + return { + id: job._id, + dataset: job.dataset ?? 'assets', + characterId: job.characterId, + status: job.status, + attempts: job.attempts ?? 0, + dueAt: job.dueAt ?? job.createdAt, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + leaseExpiresAt: job.status === 'running' ? (job.leaseExpiresAt ?? null) : null, + cancelRequested: job.cancelRequested === true, + ...(error === undefined ? {} : { error }), + }; +} + +function safeJobError(): string { return 'Asset synchronization failed'; } +function activeJobStatus(status: string): boolean { return status === 'queued' || status === 'running'; } + +async function findJob(ctx: any, jobId: string): Promise { + return await ctx.db.get(jobId as any); +} + +async function markTerminalSyncFailure(ctx: any, job: any, error = safeSyncError('terminal failure')): Promise { + if (job.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error }); + } + const character = await findCharacter(ctx, job.userId, job.characterId); + if (!character) return; + const assetWasSyncing = character.syncGeneration === job.generation || character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && (entry.status === 'syncing' || entry.status === 'queued')); + if (!assetWasSyncing) return; + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' + ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error } + : entry); + // Never clear activeGeneration/syncedAt/expiresAt: the last complete snapshot + // remains the only readable publication until a replacement succeeds. + await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error }); +} + +async function clearMatchingSyncPointer(ctx: any, character: any, generation: string, error = 'Authorization is no longer active'): Promise { + if (!character || character.syncGeneration !== generation) return false; + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' + ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error, generation: character.activeGeneration ?? undefined } + : entry); + await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error }); + return true; +} + +async function invalidateCharacterJobs(ctx: any, userId: string, characterId: number, timestamp: number): Promise { + const character = await findCharacter(ctx, userId, characterId); + const jobs = await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', userId)).collect(); + let staleSyncState = false; + for (const job of jobs) { + if (workerJob(job) && job.characterId === characterId && activeJobStatus(job.status)) { + if (job.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', userId).eq('characterId', characterId).eq('generation', job.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' }); + } + if (character && job.generation !== undefined && character.syncGeneration === job.generation) staleSyncState = true; + await ctx.db.patch(job._id, { + status: 'quarantined', updatedAt: timestamp, leaseExpiresAt: undefined, + leaseToken: undefined, workerId: undefined, cancelRequested: undefined, + lastError: 'Authorization is no longer active', finishedAt: timestamp, + }); + } + } + if (character && staleSyncState && character.syncGeneration !== undefined) await clearMatchingSyncPointer(ctx, character, character.syncGeneration); +} + +/** Server-derived enqueue. The caller supplies a session proof, never a job authority. */ +export const bridgeEnqueueAssetSync = mutationGeneric({ + args: { + ...bridgeArgs, + userId: v.string(), + characterId: v.number(), + initiatingSessionHash: v.string(), + }, + handler: async (ctx, args) => { + assertBridge(args.bridgeSecret); + assertDeploymentCapability('asset_sync'); + const timestamp = now(); + const initiating = await findSessionByHash(ctx, args.initiatingSessionHash); + if (!isUsableSession(initiating, timestamp) || initiating.userId !== args.userId) throw new Error('Session is no longer valid'); + const character = await findCharacter(ctx, args.userId, args.characterId); + if (!character || character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) { + throw new Error('Asset authorization is not active'); + } + const dedupeKey = `${args.userId}:character:${args.characterId}:assets`; + const existing = await ctx.db.query('jobs').withIndex('by_dedupe', (q: any) => q.eq('dedupeKey', dedupeKey)).collect(); + const active = existing.find((job: any) => activeJobStatus(job.status)); + if (active) return safeSyncJob(active); + const queuedJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job)); + const runningJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'running')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job)); + if (queuedJobs.length + runningJobs.length >= MAX_USER_ACTIVE_SYNC_JOBS) throw new Error('Synchronization queue is full'); + const maxAttempts = 5; + const job = await ctx.db.insert('jobs', { + userId: args.userId, + kind: 'asset_sync', + dataset: 'assets', + status: 'queued', + characterId: args.characterId, + dedupeKey, + createdAt: timestamp, + updatedAt: timestamp, + dueAt: timestamp, + attempts: 0, + maxAttempts, + priority: 0, + ownerKind: 'user', + authority: 'user', + grantRevision: character.grantRevision, + authorizationEpoch: character.authorizationEpoch, + credentialVersion: character.credentialVersion, + scopes: [...character.scopes], + cooldownGroup: SYNC_PROVIDER_GROUP, + cancelRequested: false, + }); + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' + ? { ...entry, status: 'queued', grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion, error: undefined } + : entry); + await ctx.db.patch(character._id, { state: 'queued', datasetStatuses: statuses, error: undefined }); + return safeSyncJob({ _id: job, dataset: 'assets', characterId: args.characterId, status: 'queued', attempts: 0, dueAt: timestamp, createdAt: timestamp, updatedAt: timestamp, cancelRequested: false }); + }, +}); + +export const bridgeSyncJob = queryGeneric({ + args: { ...bridgeArgs, userId: v.string(), jobId: v.string() }, + handler: async (ctx, args) => { + assertBridge(args.bridgeSecret); + const job = await findJob(ctx, args.jobId); + if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null; + return safeSyncJob(job); + }, +}); + +/** Cancellation is authenticated against the initiating session and fences a running lease immediately. */ +export const bridgeCancelSyncJob = mutationGeneric({ + args: { ...bridgeArgs, userId: v.string(), jobId: v.string(), initiatingSessionHash: v.string() }, + handler: async (ctx, args) => { + assertBridge(args.bridgeSecret); + assertDeploymentCapability('asset_sync'); + const initiating = await findSessionByHash(ctx, args.initiatingSessionHash); + if (!isUsableSession(initiating) || initiating.userId !== args.userId) throw new Error('Session is no longer valid'); + const job = await findJob(ctx, args.jobId); + if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null; + if (!activeJobStatus(job.status)) return safeSyncJob(job); + const timestamp = now(); + const character = job.characterId === undefined ? null : await findCharacter(ctx, args.userId, job.characterId); + if (job.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', args.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Synchronization cancelled' }); + } + await ctx.db.patch(job._id, { + status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, + leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, + cancelRequested: true, lastError: 'Synchronization cancelled', + }); + if (character && character.syncGeneration === job.generation) { + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' + ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' } + : entry); + await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Synchronization cancelled' }); + } else if (character && character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && entry.status === 'queued')) { + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' } : entry); + await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, error: 'Synchronization cancelled' }); + } + return safeSyncJob({ ...job, status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, cancelRequested: true, lastError: 'Synchronization cancelled' }); + }, +}); + +/** + * Claim and recover in one transaction. Expired leases are requeued before a + * candidate is selected, making process termination recoverable without a + * second scheduler or an in-memory queue. + */ +export const bridgeClaimSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, workerId: v.string(), leaseMs: v.optional(v.number()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + if (!args.workerId || args.workerId.length > 160) throw new Error('Invalid worker identity'); + const timestamp = now(); + const leaseMs = args.leaseMs ?? 10 * 60 * 1000; + if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) throw new Error('Invalid sync lease duration'); + const expired = await ctx.db.query('jobs').withIndex('by_lease_expires_at', (q: any) => q.lte('leaseExpiresAt', timestamp)).take(100); + for (const job of expired) { + if (!workerJob(job) || job.status !== 'running' || (job.leaseExpiresAt ?? Infinity) > timestamp) continue; + if (job.cancelRequested === true) { + await ctx.db.patch(job._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization cancelled' }); + } else if ((job.attempts ?? 0) >= (job.maxAttempts ?? 5)) { + const character = await findCharacter(ctx, job.userId, job.characterId); + if (!authorityMatches(character, job)) { + if (job.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' }); + await clearMatchingSyncPointer(ctx, character, job.generation); + } + await ctx.db.patch(job._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Authorization is no longer active' }); + continue; + } + await markTerminalSyncFailure(ctx, job); + await ctx.db.patch(job._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() }); + } else { + await ctx.db.patch(job._id, { status: 'queued', updatedAt: timestamp, dueAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined }); + } + } + const candidates = await ctx.db.query('jobs').withIndex('by_asset_sync_queue', (q: any) => q.eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued').lte('dueAt', timestamp)).order('asc').take(100); + if (candidates.length === 0) return null; + // Bounded fair selection: an account with an in-flight job loses to an + // account with no in-flight work; due/creation order breaks ties. + const supportedCandidates = candidates.filter((candidate: any) => workerJob(candidate)); + if (supportedCandidates.length === 0) return null; + const runningByUser = new Map(); + for (const candidate of supportedCandidates) { + if (runningByUser.has(candidate.userId)) continue; + const running = (await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', candidate.userId).eq('status', 'running')).take(20)).filter((job: any) => workerJob(job)); + runningByUser.set(candidate.userId, running.length); + } + const ordered = [...supportedCandidates].sort((left: any, right: any) => + (runningByUser.get(left.userId)! - runningByUser.get(right.userId)!) + || ((left.priority ?? 0) - (right.priority ?? 0)) + || ((left.dueAt ?? left.createdAt) - (right.dueAt ?? right.createdAt)) + || ((left.lastClaimedAt ?? 0) - (right.lastClaimedAt ?? 0)) + || String(left._id).localeCompare(String(right._id))); + for (const candidate of ordered) { + if (candidate.cancelRequested === true) { + await ctx.db.patch(candidate._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Synchronization cancelled' }); + continue; + } + if (candidate.cooldownGroup) { + const shared = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${candidate.cooldownGroup}`)).first(); + if (shared && shared.until > timestamp) continue; + } + const character = candidate.characterId === undefined ? null : await findCharacter(ctx, candidate.userId, candidate.characterId); + const authorityCurrent = authorityMatches(character, candidate); + if (!authorityCurrent) { + if (candidate.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', candidate.userId).eq('characterId', candidate.characterId).eq('generation', candidate.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' }); + await clearMatchingSyncPointer(ctx, character, candidate.generation); + } + await ctx.db.patch(candidate._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Authorization is no longer active' }); + continue; + } + const leaseToken = newIdentifier('lease'); + const attempts = (candidate.attempts ?? 0) + 1; + await ctx.db.patch(candidate._id, { status: 'running', attempts, updatedAt: timestamp, startedAt: candidate.startedAt ?? timestamp, dueAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken, workerId: args.workerId, lastClaimedAt: timestamp }); + return { ...safeSyncJob({ ...candidate, status: 'running', attempts, updatedAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken }), jobId: candidate._id, leaseToken, userId: candidate.userId, characterId: candidate.characterId, grantRevision: candidate.grantRevision, authorizationEpoch: candidate.authorizationEpoch, credentialVersion: candidate.credentialVersion, scopes: candidate.scopes ?? [], ...(candidate.generation === undefined ? {} : { generation: candidate.generation }), cooldownGroup: candidate.cooldownGroup ?? SYNC_PROVIDER_GROUP }; + } + return null; + }, +}); + +export const bridgeRenewSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), workerId: v.string(), leaseMs: v.optional(v.number()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const job = await findJob(ctx, args.jobId); + if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== args.leaseToken || job.workerId !== args.workerId) return false; + if (job.cancelRequested === true || (job.leaseExpiresAt ?? 0) <= now()) return false; + const leaseMs = args.leaseMs ?? 10 * 60 * 1000; + if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) return false; + await ctx.db.patch(job._id, { leaseExpiresAt: now() + leaseMs, updatedAt: now() }); + return true; + }, +}); + +async function jobLease(ctx: any, jobId: string, leaseToken: string): Promise { + const job = await findJob(ctx, jobId); + if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== leaseToken || (job.leaseExpiresAt ?? 0) <= now()) throw new Error('Synchronization lease is not valid'); + return job; +} + +async function jobCharacter(ctx: any, job: any): Promise { + const character = await findCharacter(ctx, job.userId, job.characterId); + if (!authorityMatches(character, job)) { + throw new Error('Synchronization authorization is stale'); + } + return character; +} + +export const bridgeRefreshSyncJobAuthority = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), expectedGrantRevision: v.number(), expectedAuthorizationEpoch: v.number(), expectedCredentialVersion: v.number(), expectedScopes: v.array(v.string()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const job = await jobLease(ctx, args.jobId, args.leaseToken); + const character = await findCharacter(ctx, job.userId, job.characterId); + const expected = { grantRevision: args.expectedGrantRevision, authorizationEpoch: args.expectedAuthorizationEpoch, credentialVersion: args.expectedCredentialVersion, scopes: args.expectedScopes }; + if (!authorityMatches(character, expected)) throw new Error('Synchronization authorization changed'); + if (!authorityMatches(character, job)) throw new Error('Synchronization job authority is stale'); + return expected; + }, +}); + +export const bridgeRestartSyncJobGeneration = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const job = await jobLease(ctx, args.jobId, args.leaseToken); + if (job.generation === undefined) return null; + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization refreshed before publication' }); + const character = await findCharacter(ctx, job.userId, job.characterId); + if (character?.syncGeneration === job.generation) await ctx.db.patch(character._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined, state: 'queued' }); + await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() }); + return null; + }, +}); + +export const bridgeBeginSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const job = await jobLease(ctx, args.jobId, args.leaseToken); + let character = await jobCharacter(ctx, job); + if (job.generation !== undefined) { + const existing = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (existing && existing.status === 'staging' && existing.grantRevision === job.grantRevision && existing.authorizationEpoch === job.authorizationEpoch && existing.credentialVersion === job.credentialVersion) return job.generation; + if (existing?.status === 'staging') await ctx.db.patch(existing._id, { status: 'failed', error: 'Synchronization authority changed' }); + await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() }); + const oldCharacter = await findCharacter(ctx, job.userId, job.characterId); + if (oldCharacter?.syncGeneration === job.generation) await ctx.db.patch(oldCharacter._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined }); + // The character read above predates the atomic cleanup. Re-read it before + // checking the lease, otherwise a reclaimed job sees its own old pointer + // and incorrectly reports an active synchronization. + character = await findCharacter(ctx, job.userId, job.characterId); + if (!character || !authorityMatches(character, job)) throw new Error('Synchronization authorization is stale'); + } + if (character.syncGeneration && (character.syncLeaseExpiresAt ?? 0) > now()) throw new Error('Character synchronization is already leased'); + const generation = newIdentifier('generation'); + await ctx.db.insert('assetGenerations', { userId: job.userId, characterId: job.characterId, generation, status: 'staging', stagedChunkCount: 0, grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion }); + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'syncing', grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion, error: undefined } : entry); + await ctx.db.patch(character._id, { state: 'syncing', datasetStatuses: statuses, syncGeneration: generation, syncLeaseExpiresAt: now() + 10 * 60 * 1000, error: undefined }); + await ctx.db.patch(job._id, { generation, updatedAt: now() }); + return generation; + }, +}); + +export const bridgeStageSyncAssets = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), index: v.number(), assets: v.array(assetValidator) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + if (!Number.isInteger(args.index) || args.index < 0 || args.assets.length > MAX_ASSET_CHUNK) throw new Error('Invalid asset chunk'); + const job = await jobLease(ctx, args.jobId, args.leaseToken); + const character = await jobCharacter(ctx, job); + if (args.assets.some((asset: any) => asset.characterId !== job.characterId)) throw new Error('Asset character mismatch'); + if (!job.generation || character.syncGeneration !== job.generation) throw new Error('Synchronization generation is not current'); + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first(); + if (!generation || generation.status !== 'staging') throw new Error('Snapshot is not staging'); + const stagedChunkCount = generation.stagedChunkCount ?? 0; + if (stagedChunkCount >= MAX_CHUNKS || args.index > stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order'); + const existing = await ctx.db.query('assetChunks').withIndex('by_snapshot_index', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation).eq('index', args.index)).first(); + if (existing) { + // Convex may commit a mutation while its network response is lost. A + // duplicate delivery is accepted only when its payload is identical. + if (JSON.stringify(existing.assets) !== JSON.stringify(args.assets)) throw new Error('Asset chunk already exists'); + return null; + } + if (args.index !== stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order'); + await ctx.db.insert('assetChunks', { userId: job.userId, characterId: job.characterId, generation: job.generation, index: args.index, assets: args.assets }); + await ctx.db.patch(generation._id, { stagedChunkCount: stagedChunkCount + 1 }); + return null; + }, +}); + +function validJobSnapshotTimes(syncedAt: number, expiresAt: number, timestamp: number): boolean { + return Number.isFinite(syncedAt) && syncedAt >= 0 && syncedAt <= timestamp && Number.isFinite(expiresAt) && expiresAt > timestamp && expiresAt > syncedAt && expiresAt - syncedAt <= SNAPSHOT_MAX_LIFETIME_MS; +} + +export const bridgeCommitSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), chunkCount: v.number(), syncedAt: v.number(), expiresAt: v.number() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const job = await findJob(ctx, args.jobId); + if (job?.status === 'succeeded' && job.generation !== undefined) return true; + const leased = await jobLease(ctx, args.jobId, args.leaseToken); + const character = await jobCharacter(ctx, leased); + if (!leased.generation) throw new Error('Synchronization generation is not current'); + if (!Number.isInteger(args.chunkCount) || args.chunkCount < 0 || args.chunkCount > MAX_CHUNKS || !validJobSnapshotTimes(args.syncedAt, args.expiresAt, now())) throw new Error('Invalid snapshot'); + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first(); + if (!generation || generation.status !== 'staging' || (generation.stagedChunkCount ?? 0) !== args.chunkCount) throw new Error('Snapshot is incomplete'); + if (generation.grantRevision !== character.grantRevision || generation.authorizationEpoch !== character.authorizationEpoch || generation.credentialVersion !== character.credentialVersion) throw new Error('Synchronization authorization is stale'); + await ctx.db.patch(generation._id, { status: 'complete', chunkCount: args.chunkCount, syncedAt: args.syncedAt, expiresAt: args.expiresAt }); + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'ready', generation: leased.generation, syncedAt: args.syncedAt, error: undefined } : entry); + await ctx.db.patch(character._id, { state: 'ready', datasetStatuses: statuses, syncedAt: args.syncedAt, expiresAt: args.expiresAt, activeGeneration: leased.generation, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: undefined }); + await ctx.db.patch(leased._id, { status: 'succeeded', updatedAt: now(), finishedAt: now(), leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, cancelRequested: undefined, lastError: undefined }); + return true; + }, +}); + +export const bridgeFailSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const leased = await jobLease(ctx, args.jobId, args.leaseToken); + const character = await jobCharacter(ctx, leased); + const timestamp = now(); + if (leased.generation !== undefined) { + const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first(); + if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: safeJobError() }); + } + const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: safeJobError() } : entry); + await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, error: safeJobError(), syncGeneration: undefined, syncLeaseExpiresAt: undefined }); + await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() }); + return true; + }, +}); + +/** Reschedule transient provider/cooldown failures without exposing their raw text. */ +export const bridgeDeferSyncJob = mutationGeneric({ + args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), dueAt: v.number(), cooldownContext: v.optional(v.string()), cooldownUntil: v.optional(v.number()), cooldownGroup: v.optional(v.string()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const leased = await jobLease(ctx, args.jobId, args.leaseToken); + const timestamp = now(); + if (!Number.isFinite(args.dueAt) || args.dueAt < timestamp || args.dueAt > timestamp + 24 * 60 * 60 * 1000) throw new Error('Invalid retry due time'); + if ((leased.attempts ?? 0) >= (leased.maxAttempts ?? 5)) { + await jobCharacter(ctx, leased); + await markTerminalSyncFailure(ctx, leased); + await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() }); + return false; + } + await ctx.db.patch(leased._id, { status: 'queued', updatedAt: timestamp, dueAt: args.dueAt, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization scheduled for retry', ...(args.cooldownGroup ? { cooldownGroup: args.cooldownGroup } : {}) }); + if (args.cooldownContext && args.cooldownUntil !== undefined && Number.isFinite(args.cooldownUntil)) { + const contexts = [args.cooldownContext, ...(args.cooldownGroup ? [`group:${args.cooldownGroup}`] : [])]; + for (const context of contexts) { + const existing = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first(); + const until = Math.max(existing?.until ?? 0, args.cooldownUntil); + if (existing) await ctx.db.patch(existing._id, { until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp }); + else await ctx.db.insert('syncCooldowns', { context, until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp }); + } + } + return true; + }, +}); + +async function currentCooldown(ctx: any, context: string, group?: string): Promise { + const rows = [await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first()]; + if (group) rows.push(await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${group}`)).first()); + const row = rows.filter(Boolean).sort((left: any, right: any) => right.until - left.until)[0]; + return row && row.until > now() ? { until: row.until, group: row.group, kind: row.kind } : null; +} + +export const bridgeCooldown = queryGeneric({ + args: { ...bridgeArgs, context: v.string() }, + handler: async (ctx, args) => { + assertBridge(args.bridgeSecret); + return await currentCooldown(ctx, args.context); + }, +}); + +export const bridgeSyncWorkerCooldown = queryGeneric({ + args: { ...workerBridgeArgs, context: v.string(), group: v.optional(v.string()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + return await currentCooldown(ctx, args.context, args.group); + }, +}); + +async function markRefreshUncertainRecord(ctx: any, userId: string, characterId: number, lease: any, character: any, record: any): Promise { + if (!lease || !character || !record || record.version !== lease.expectedVersion) { + if (lease) await ctx.db.delete(lease._id); + return false; + } + const epoch = character.authorizationEpoch + 1; + const statuses = DATASET_NAMES.map((dataset) => ({ dataset, status: 'blocked' as const, grantRevision: character.grantRevision, authorizationEpoch: epoch, credentialVersion: character.credentialVersion, invalidatedAt: now(), error: 'Reconnect required' })); + // Retain the last complete generation metadata; authorization status blocks + // reads until reconnect, while a later reconnect can explicitly replace it. + await ctx.db.patch(character._id, { authorizationStatus: 'reconnect_required', authorizationEpoch: epoch, datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Reconnect required', state: 'error' }); + await ctx.db.delete(record._id); await ctx.db.delete(lease._id); + await invalidateCharacterJobs(ctx, userId, characterId, now()); + return true; +} + +export const bridgeMarkRefreshDispatched = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false; + await ctx.db.patch(lease._id, { dispatchStartedAt: now() }); + return true; + }, +}); + +/** Durable per-character refresh serialization. The encrypted grant never reaches browsers. */ +export const bridgeAcquireRefreshLease = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), ownerId: v.string(), leaseMs: v.optional(v.number()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const character = await findCharacter(ctx, args.userId, args.characterId); + if (!character || character.authorizationStatus !== 'active') throw new Error('Character authorization is not active'); + const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!record) throw new Error('Character authorization is missing'); + const timestamp = now(); const leaseMs = args.leaseMs ?? 2 * 60 * 1000; + if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) throw new Error('Invalid refresh lease duration'); + const existing = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (existing && existing.expiresAt > timestamp) return null; + if (existing?.dispatchStartedAt !== undefined) { + await markRefreshUncertainRecord(ctx, args.userId, args.characterId, existing, character, record); + return null; + } + if (existing) await ctx.db.delete(existing._id); + const leaseToken = newIdentifier('refresh'); + await ctx.db.insert('refreshLeases', { userId: args.userId, characterId: args.characterId, leaseToken, ownerId: args.ownerId, expiresAt: timestamp + leaseMs, expectedVersion: record.version }); + return { leaseToken, expiresAt: timestamp + leaseMs, expectedVersion: record.version, encryptedTokens: record.encryptedTokens, tokenExpiresAt: record.tokenExpiresAt, scopes: record.scopes, grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion }; + }, +}); + +export const bridgeRenewRefreshLease = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), leaseMs: v.optional(v.number()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now()) return false; + const leaseMs = args.leaseMs ?? 2 * 60 * 1000; + if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) return false; + await ctx.db.patch(lease._id, { expiresAt: now() + leaseMs }); return true; + }, +}); + +export const bridgeReleaseRefreshLease = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false; + await ctx.db.delete(lease._id); + return true; + }, +}); + +export const bridgeCommitRefresh = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number(), encryptedTokens: v.string(), tokenExpiresAt: v.number(), scopes: v.array(v.string()), jobId: v.optional(v.string()), jobLeaseToken: v.optional(v.string()) }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + const character = await findCharacter(ctx, args.userId, args.characterId); + const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt === undefined || !character || character.authorizationStatus !== 'active' || !record || record.version !== args.expectedVersion) return null; + if (!validActualScopes(args.scopes) || !scopesSubset(args.scopes, character.scopes ?? [])) throw new Error('Invalid refreshed scopes'); + const nextVersion = record.version + 1; + const nextRevision = character.grantRevision + (scopesEqual(character.scopes, args.scopes) ? 0 : 1); + const updated = updateDatasetStatuses(character, args.scopes, nextRevision, character.authorizationEpoch, nextVersion, now()); + await ctx.db.patch(record._id, { encryptedTokens: args.encryptedTokens, tokenExpiresAt: args.tokenExpiresAt, scopes: args.scopes, version: nextVersion }); + await ctx.db.patch(character._id, { scopes: args.scopes, grantRevision: nextRevision, credentialVersion: nextVersion, datasetStatuses: updated.statuses, ...(updated.assetsInvalidated ? { state: 'error' as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}) }); + if (updated.assetsInvalidated) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now()); + else if (args.jobId && args.jobLeaseToken) { + const job = await findJob(ctx, args.jobId); + if (job && job.userId === args.userId && job.characterId === args.characterId && job.status === 'running' && job.leaseToken === args.jobLeaseToken) { + await ctx.db.patch(job._id, { grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: nextVersion, scopes: args.scopes, updatedAt: now() }); + } + } + await ctx.db.delete(lease._id); + return { saved: true, credentialVersion: nextVersion, grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, scopes: args.scopes }; + }, +}); + +/** Unknown rotating-token outcomes fail closed and require an explicit reconnect. */ +export const bridgeMarkRefreshUncertain = mutationGeneric({ + args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number() }, + handler: async (ctx, args) => { + assertSyncWorker(args.workerSecret); + const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId) return false; + const character = await findCharacter(ctx, args.userId, args.characterId); + const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first(); + if (lease.expectedVersion !== args.expectedVersion) return false; + return await markRefreshUncertainRecord(ctx, args.userId, args.characterId, lease, character, record); + }, +}); + export const bridgeSaveInteraction = mutationGeneric({ args: { ...bridgeArgs, diff --git a/src/convex/schema.ts b/src/convex/schema.ts index 908f35f..757f849 100644 --- a/src/convex/schema.ts +++ b/src/convex/schema.ts @@ -4,6 +4,7 @@ import { v } from "convex/values"; /** The persisted character state is deliberately a small, explicit state machine. */ export const characterStateValidator = v.union( v.literal("never"), + v.literal("queued"), v.literal("syncing"), v.literal("ready"), v.literal("error"), @@ -16,6 +17,7 @@ export const authorizationStatusValidator = v.union( v.literal("quarantined"), v.literal("revoked"), v.literal("transferred"), + v.literal("reconnect_required"), ); export const datasetStateValidator = v.union( v.literal("never"), @@ -180,9 +182,56 @@ export default defineSchema({ updatedAt: v.number(), leaseExpiresAt: v.optional(v.number()), characterId: v.optional(v.number()), + // Durable sync orchestration fields. They are optional for pre-T07 + // placeholder rows; new jobs always populate the complete fence. + dataset: v.optional(v.string()), + dedupeKey: v.optional(v.string()), + dueAt: v.optional(v.number()), + attempts: v.optional(v.number()), + maxAttempts: v.optional(v.number()), + priority: v.optional(v.number()), + lastClaimedAt: v.optional(v.number()), + leaseToken: v.optional(v.string()), + workerId: v.optional(v.string()), + cancelRequested: v.optional(v.boolean()), + cancelledAt: v.optional(v.number()), + startedAt: v.optional(v.number()), + finishedAt: v.optional(v.number()), + lastError: v.optional(v.string()), + generation: v.optional(v.string()), + grantRevision: v.optional(v.number()), + authorizationEpoch: v.optional(v.number()), + credentialVersion: v.optional(v.number()), + scopes: v.optional(v.array(v.string())), + ownerKind: v.optional(v.string()), + authority: v.optional(v.string()), + cooldownGroup: v.optional(v.string()), }) .index("by_user_status", ["userId", "status"]) - .index("by_lease_expires_at", ["leaseExpiresAt"]), + .index("by_lease_expires_at", ["leaseExpiresAt"]) + .index("by_dedupe", ["dedupeKey"]) + .index("by_status_due", ["status", "dueAt"]) + .index("by_kind_status_due", ["kind", "status", "dueAt"]) + .index("by_asset_sync_queue", ["kind", "dataset", "authority", "ownerKind", "status", "dueAt"]) + .index("by_user_asset_status", ["userId", "kind", "dataset", "authority", "ownerKind", "status"]), + + refreshLeases: defineTable({ + userId: v.string(), + characterId: v.number(), + leaseToken: v.string(), + ownerId: v.string(), + expiresAt: v.number(), + expectedVersion: v.number(), + dispatchStartedAt: v.optional(v.number()), + }).index("by_user_character", ["userId", "characterId"]), + + syncCooldowns: defineTable({ + context: v.string(), + until: v.number(), + group: v.optional(v.string()), + kind: v.string(), + updatedAt: v.number(), + }).index("by_context", ["context"]).index("by_until", ["until"]), referenceSyncMetadata: defineTable({ source: v.string(), diff --git a/src/lib/client/sync-queue.ts b/src/lib/client/sync-queue.ts new file mode 100644 index 0000000..5f45984 --- /dev/null +++ b/src/lib/client/sync-queue.ts @@ -0,0 +1,95 @@ +export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined'; +export type SyncJob = { + id: string; + dataset: 'assets'; + characterId: number; + status: SyncJobStatus; + attempts: number; + dueAt: number; + createdAt: number; + updatedAt: number; + leaseExpiresAt: number | null; + cancelRequested: boolean; + error?: string; +}; +export type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' }; +export type SyncEnqueueResponse = { queued: number; jobs: SyncJob[]; skipped: SyncSkip[]; partial?: boolean; demo?: boolean }; + +const JOB_STATUSES = new Set(['queued', 'running', 'succeeded', 'failed', 'cancelled', 'quarantined']); +const SKIP_REASONS = new Set(['authorization_required', 'queue_full', 'enqueue_failed']); + +function parseJob(value: unknown): SyncJob | null { + if (!value || typeof value !== 'object') return null; + const row = value as Record; + if (typeof row.id !== 'string' || typeof row.characterId !== 'number' || !Number.isSafeInteger(row.characterId) || typeof row.status !== 'string' || !JOB_STATUSES.has(row.status as SyncJobStatus)) return null; + return { + id: row.id, dataset: 'assets', characterId: row.characterId, status: row.status as SyncJobStatus, + attempts: typeof row.attempts === 'number' ? row.attempts : 0, + dueAt: typeof row.dueAt === 'number' ? row.dueAt : 0, + createdAt: typeof row.createdAt === 'number' ? row.createdAt : 0, + updatedAt: typeof row.updatedAt === 'number' ? row.updatedAt : 0, + leaseExpiresAt: typeof row.leaseExpiresAt === 'number' ? row.leaseExpiresAt : null, + cancelRequested: row.cancelRequested === true, + ...(typeof row.error === 'string' ? { error: row.error } : {}), + }; +} + +export function parseSyncEnqueueBody(value: unknown): SyncEnqueueResponse | { error: string } { + if (!value || typeof value !== 'object') return { error: 'Synchronization response was invalid.' }; + const body = value as Record; + if (!Array.isArray(body.jobs) || !Array.isArray(body.skipped)) return { error: typeof body.error === 'string' ? body.error : 'Synchronization response was invalid.' }; + const jobs = body.jobs.map(parseJob); + const skipped = body.skipped.map((entry: unknown) => { + if (!entry || typeof entry !== 'object') return null; + const row = entry as Record; + return typeof row.characterId === 'number' && Number.isSafeInteger(row.characterId) && typeof row.reason === 'string' && SKIP_REASONS.has(row.reason as SyncSkip['reason']) + ? { characterId: row.characterId, reason: row.reason as SyncSkip['reason'] } : null; + }); + if (jobs.some((job) => job === null) || skipped.some((entry) => entry === null)) return { error: 'Synchronization response was invalid.' }; + const queued = typeof body.queued === 'number' && Number.isSafeInteger(body.queued) && body.queued >= 0 ? body.queued : jobs.length; + return { queued, jobs: jobs as SyncJob[], skipped: skipped as SyncSkip[], ...(body.partial === true ? { partial: true } : {}), ...(body.demo === true ? { demo: true } : {}) }; +} + +export async function enqueueSync(signal?: AbortSignal): Promise<{ response: Response; body: SyncEnqueueResponse | { error: string } }> { + const response = await fetch('/api/sync', signal ? { method: 'POST', signal } : { method: 'POST' }); + return { response, body: parseSyncEnqueueBody(await response.json().catch(() => null)) }; +} + +export const SYNC_STATUS_TIMEOUT_MS = 15_000; + +export async function syncJobStatus(id: string, signal?: AbortSignal, timeoutMs = SYNC_STATUS_TIMEOUT_MS): Promise { + const requestController = new AbortController(); + let timeout: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const aborted = new Promise((_, reject) => { + onAbort = () => { requestController.abort(); reject(new DOMException('The operation was aborted', 'AbortError')); }; + if (signal?.aborted) onAbort(); + else signal?.addEventListener('abort', onAbort, { once: true }); + }); + const timedOut = new Promise((_, reject) => { + timeout = setTimeout(() => { requestController.abort(); reject(new Error('Synchronization status timed out.')); }, timeoutMs); + }); + try { + const response = await Promise.race([fetch(`/api/sync/status?job=${encodeURIComponent(id)}`, { signal: requestController.signal }), timedOut, aborted]); + if (response.status === 404) return null; + if (!response.ok) throw new Error('Synchronization status unavailable.'); + return await response.json() as SyncJob; + } finally { + if (timeout !== undefined) clearTimeout(timeout); + if (signal && onAbort) signal.removeEventListener('abort', onAbort); + } +} + +export async function cancelSyncJob(id: string): Promise { + const response = await fetch('/api/sync/cancel', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jobId: id }), + }); + if (!response.ok) throw new Error('Could not cancel synchronization.'); + return await response.json() as SyncJob; +} + +export function syncTerminal(status: SyncJobStatus): boolean { + return status === 'succeeded' || status === 'failed' || status === 'cancelled' || status === 'quarantined'; +} diff --git a/src/lib/domain/model.ts b/src/lib/domain/model.ts index 2aee212..23dcd45 100644 --- a/src/lib/domain/model.ts +++ b/src/lib/domain/model.ts @@ -18,12 +18,12 @@ export type Character = { scopes: string[]; /** Exactly one active primary may authenticate while signed out. */ loginRole: 'primary' | 'data_only'; - authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred'; + authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred' | 'reconnect_required'; grantRevision: number; credentialVersion: number; authorizationEpoch: number; datasets: CharacterDatasetStatus[]; - assets: { status: 'never' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string }; + assets: { status: 'never' | 'queued' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string }; }; export type Asset = { itemId: string; @@ -307,6 +307,20 @@ export type JobOwner = export type JobAuthority = 'user' | 'system'; export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined'; +export type SyncJobDataset = 'assets'; +export type SafeSyncJob = { + id: string; + dataset: SyncJobDataset; + characterId: number; + status: SyncJobStatus; + attempts: number; + dueAt: number; + createdAt: number; + updatedAt: number; + leaseExpiresAt: number | null; + cancelRequested: boolean; + error?: string; +}; export type PublicationOutcome = 'success' | 'failure'; export type PublicationResourceFence = SelectedResource & { /** Monotonic revocation/quarantine epoch, distinct from grant and credential versions. */ diff --git a/src/lib/server/application.ts b/src/lib/server/application.ts index 8565240..24ab873 100644 --- a/src/lib/server/application.ts +++ b/src/lib/server/application.ts @@ -1,10 +1,10 @@ import type { RequestEvent } from '@sveltejs/kit'; -import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository } from '$lib/domain/model'; +import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository, SafeSyncJob } from '$lib/domain/model'; import { AssetService } from '$lib/domain/assets'; import { configuration, requireLiveConfig } from './config'; import { ConvexStore } from './store'; import { TokenCipher, hashSecret } from './security'; -import { EveSso, type VerifiedGrant } from './sso'; +import { EveSso } from './sso'; import { EsiClient, EsiMarketEstimates } from './esi'; import { SqliteReferenceRepository } from './reference'; import { capabilityDisabledReason, capabilityEnabled, type ReleaseCapability } from './release-gate'; @@ -91,104 +91,29 @@ export async function assetReport( finally { reference?.close(); } } -type CommittedGrant = VerifiedGrant & { - credentialVersion: number; - grantRevision: number; - authorizationEpoch: number; -}; -const refreshes = new Map>(); -async function characterGrant(userId: string, characterId: number, revalidateOwnership = false): Promise { - const key = `${userId}:${characterId}`; - const existing = refreshes.get(key); if (existing) return existing; - const promise = (async () => { - const { store, cipher, sso } = live(); - const record = await store.tokenRecord(userId, characterId); - if (!record) throw new Error('Character authorization is missing'); - const previous = await cipher.open(record.encryptedTokens, `character:${characterId}`); - if (previous.characterId !== characterId) throw new Error('Character authorization mismatch'); - if (!revalidateOwnership && previous.expiresAt > Date.now() + 60_000) { - const current = (await store.characters(userId)).find((entry) => entry.id === characterId); - if (!current) throw new Error('Character authorization is missing'); - return { - ...previous, - credentialVersion: current.credentialVersion, - grantRevision: current.grantRevision, - authorizationEpoch: current.authorizationEpoch, - }; - } - const grant = await sso.refresh(previous.refreshToken); - if (grant.characterId !== characterId || grant.ownerHash !== previous.ownerHash) { - await store.quarantineAuthorization(userId, characterId).catch(() => {}); - throw new Error('Character ownership changed; reconnect required'); - } - const saved = await store.saveTokens(userId, characterId, { encryptedTokens: await cipher.seal(grant, `character:${characterId}`), tokenExpiresAt: grant.expiresAt, scopes: grant.scopes, expectedVersion: record.version }); - if (!saved) throw new Error('Authorization changed concurrently; retry synchronization'); - return { - ...grant, - credentialVersion: saved.credentialVersion, - grantRevision: saved.grantRevision, - authorizationEpoch: saved.authorizationEpoch, - scopes: saved.scopes, - }; +type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' }; - })(); - refreshes.set(key, promise); - try { return await promise; } finally { if (refreshes.get(key) === promise) refreshes.delete(key); } -} -export async function synchronize(userId: string) { +export async function synchronize(userId: string, initiatingSessionHash?: string) { const config = configuration(); if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) { throw new Error(capabilityDisabledReason('asset_sync')); } - if (config.mode === 'demo') return { completed: 2, failed: [] as number[], demo: true }; - const { store, esi } = live(); + if (config.mode === 'demo') return { queued: 2, jobs: [] as SafeSyncJob[], skipped: [] as SyncSkip[], partial: false, demo: true }; + if (!initiatingSessionHash) throw new Error('Authenticated session proof is required'); + const { store } = live(); const characters = await store.characters(userId); - const failed: number[] = []; let completed = 0; + const jobs: SafeSyncJob[] = []; + const skipped: SyncSkip[] = []; for (const character of characters) { - if (character.authorizationStatus !== 'active') { failed.push(character.id); continue; } - if (!character.scopes.includes('esi-assets.read_assets.v1')) { failed.push(character.id); continue; } - let generation: string | undefined; + if (character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) { + skipped.push({ characterId: character.id, reason: 'authorization_required' }); + continue; + } try { - // Refresh first so the generation fence records the credential version - // that actually authorizes the ESI request and publication. - const grant = await characterGrant(userId, character.id, true); - if (!grant.scopes.includes('esi-assets.read_assets.v1')) throw new Error('Asset authorization missing'); - // Refresh/CAS may have advanced authority while the initial character - // list was in memory. Reload before leasing and bind beginSync to the - // exact revisions/scopes that authorize the transport request. - const current = (await store.characters(userId)).find((entry) => entry.id === character.id); - if (!current || current.authorizationStatus !== 'active' - || !current.scopes.includes('esi-assets.read_assets.v1') - || current.credentialVersion !== grant.credentialVersion - || current.grantRevision !== grant.grantRevision - || current.authorizationEpoch !== grant.authorizationEpoch - || JSON.stringify([...current.scopes].sort()) !== JSON.stringify([...grant.scopes].sort())) { - throw new Error('Asset authorization changed after credential commit'); - } - const authority = { - userId, - principal: grant.ownerHash, - characterId: current.id, - scopes: current.scopes, - grantRevision: grant.grantRevision, - authorizationEpoch: grant.authorizationEpoch, - credentialVersion: grant.credentialVersion, - } as const; - generation = await store.beginSync(userId, current.id, { - scopes: authority.scopes, - grantRevision: authority.grantRevision, - authorizationEpoch: authority.authorizationEpoch, - credentialVersion: authority.credentialVersion, - }); - const snapshot = await esi.getAssets(current.id, grant.accessToken, authority); - const chunks = Math.ceil(snapshot.assets.length / 250); - for (let index = 0; index < chunks; index++) await store.stageAssets(userId, character.id, generation, index, snapshot.assets.slice(index * 250, (index + 1) * 250)); - await store.finishSync(userId, character.id, generation, chunks, snapshot.syncedAt, snapshot.expiresAt); - completed++; - } catch { - failed.push(character.id); - if (generation) await store.failSync(userId, character.id, generation, 'Synchronization failed. Reconnect if authorization was revoked, or retry after ESI recovers.').catch(() => {}); + jobs.push(await store.enqueueAssetSync(userId, character.id, initiatingSessionHash)); + } catch (error) { + skipped.push({ characterId: character.id, reason: error instanceof Error && error.message === 'Synchronization queue is full' ? 'queue_full' : 'enqueue_failed' }); } } - return { completed, failed, demo: false }; + return { queued: jobs.length, jobs, skipped, partial: skipped.length > 0, demo: false }; } diff --git a/src/lib/server/store.ts b/src/lib/server/store.ts index eabb0a0..63b22a0 100644 --- a/src/lib/server/store.ts +++ b/src/lib/server/store.ts @@ -5,6 +5,7 @@ import type { Character, CharacterAssetRepository, UserId, + SafeSyncJob, } from "$lib/domain/model"; export type OAuthStateRecord = { @@ -67,6 +68,31 @@ export type SavedInteraction = SaveInteractionInput & { created: boolean; }; +export type ClaimedSyncJob = SafeSyncJob & { + jobId: string; + leaseToken: string; + userId: string; + characterId: number; + grantRevision: number; + authorizationEpoch: number; + credentialVersion: number; + scopes: string[]; + generation?: string; + cooldownGroup?: string; +}; + +export type RefreshLease = { + leaseToken: string; + expiresAt: number; + expectedVersion: number; + encryptedTokens: string; + tokenExpiresAt: number; + scopes: string[]; + grantRevision: number; + authorizationEpoch: number; + credentialVersion: number; +}; + type AssetPage = { page: Asset[]; isDone: boolean; @@ -83,6 +109,9 @@ const QUERY = { tokenRecord: "functions:bridgeTokenRecord", characters: "functions:bridgeCharacters", assetsPage: "functions:bridgeAssetsPage", + syncJob: "functions:bridgeSyncJob", + cooldown: "functions:bridgeCooldown", + workerCooldown: "functions:bridgeSyncWorkerCooldown", } as const; const MUTATION = { @@ -99,6 +128,23 @@ const MUTATION = { setPrimary: "functions:bridgeSetPrimary", invalidateAuthorization: "functions:bridgeInvalidateAuthorization", quarantineAuthorization: "functions:bridgeQuarantineAuthorization", + enqueueAssetSync: "functions:bridgeEnqueueAssetSync", + cancelSyncJob: "functions:bridgeCancelSyncJob", + claimSyncJob: "functions:bridgeClaimSyncJob", + renewSyncJob: "functions:bridgeRenewSyncJob", + beginSyncJob: "functions:bridgeBeginSyncJob", + refreshSyncJobAuthority: "functions:bridgeRefreshSyncJobAuthority", + restartSyncJobGeneration: "functions:bridgeRestartSyncJobGeneration", + stageSyncAssets: "functions:bridgeStageSyncAssets", + commitSyncJob: "functions:bridgeCommitSyncJob", + failSyncJob: "functions:bridgeFailSyncJob", + deferSyncJob: "functions:bridgeDeferSyncJob", + acquireRefreshLease: "functions:bridgeAcquireRefreshLease", + renewRefreshLease: "functions:bridgeRenewRefreshLease", + releaseRefreshLease: "functions:bridgeReleaseRefreshLease", + commitRefresh: "functions:bridgeCommitRefresh", + markRefreshUncertain: "functions:bridgeMarkRefreshUncertain", + markRefreshDispatched: "functions:bridgeMarkRefreshDispatched", } as const; function queryReference(name: string): any { @@ -110,28 +156,37 @@ function mutationReference(name: string): any { } /** - * Server-side Convex repository. Every call goes through a public bridge - * function carrying the dedicated bridge secret; this adapter never uses a - * deployment admin key and is not safe to import into browser code. + * Server-side Convex repository. Web calls use the general bridge secret; + * worker lifecycle calls use a separate narrowly-scoped worker secret. This + * adapter never uses a deployment admin key and is not safe to import into + * browser code. */ export class ConvexStore implements CharacterAssetRepository { private readonly client: ConvexHttpClient; - private readonly bridgeSecret: string; + private readonly bridgeSecret?: string; + private readonly workerSecret?: string; - public constructor(url: string, bridgeSecret: string) { + public constructor(url: string, bridgeSecret?: string, workerSecret?: string) { if (!url) throw new Error("Convex URL is required"); - if (!bridgeSecret) throw new Error("Convex bridge secret is required"); + if (!bridgeSecret && !workerSecret) throw new Error("Convex bridge or worker secret is required"); + this.bridgeSecret = bridgeSecret; + this.workerSecret = workerSecret; this.client = new ConvexHttpClient(url, { skipConvexDeploymentUrlCheck: true, logger: false, }); - this.bridgeSecret = bridgeSecret; } private bridgeArgs(args: Record): Record { + if (!this.bridgeSecret) throw new Error("General bridge authority is unavailable to this store"); return { ...args, bridgeSecret: this.bridgeSecret }; } + private workerArgs(args: Record): Record { + if (!this.workerSecret) throw new Error("Worker bridge authority is unavailable to this store"); + return { ...args, workerSecret: this.workerSecret }; + } + private async query(name: string, args: Record): Promise { return (await this.client.query( queryReference(name), @@ -147,6 +202,21 @@ export class ConvexStore implements CharacterAssetRepository { )) as T; } + private async workerMutation(name: string, args: Record): Promise { + return (await this.client.mutation( + mutationReference(name), + this.workerArgs(args), + { skipQueue: true }, + )) as T; + } + + private async workerQuery(name: string, args: Record): Promise { + return (await this.client.query( + queryReference(name), + this.workerArgs(args), + )) as T; + } + public async session(hash: string): Promise<{ userId: string; sessionId: string } | null> { return await this.query<{ userId: string; sessionId: string } | null>(QUERY.session, { sessionHash: hash, @@ -294,6 +364,82 @@ export class ConvexStore implements CharacterAssetRepository { throw new Error(SNAPSHOT_CHANGED); } + public async enqueueAssetSync(userId: string, characterId: number, initiatingSessionHash: string): Promise { + return await this.mutation(MUTATION.enqueueAssetSync, { userId, characterId, initiatingSessionHash }); + } + + public async syncJob(userId: string, jobId: string): Promise { + return await this.query(QUERY.syncJob, { userId, jobId }); + } + + public async cancelSyncJob(userId: string, jobId: string, initiatingSessionHash: string): Promise { + return await this.mutation(MUTATION.cancelSyncJob, { userId, jobId, initiatingSessionHash }); + } + + public async claimSyncJob(workerId: string, leaseMs?: number): Promise { + return await this.workerMutation(MUTATION.claimSyncJob, { workerId, ...(leaseMs === undefined ? {} : { leaseMs }) }); + } + + public async renewSyncJob(jobId: string, leaseToken: string, workerId: string, leaseMs?: number): Promise { + return await this.workerMutation(MUTATION.renewSyncJob, { jobId, leaseToken, workerId, ...(leaseMs === undefined ? {} : { leaseMs }) }); + } + + public async beginSyncJob(jobId: string, leaseToken: string): Promise { + return await this.workerMutation(MUTATION.beginSyncJob, { jobId, leaseToken }); + } + + public async refreshSyncJobAuthority(jobId: string, leaseToken: string, expected: { grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }): Promise<{ grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }> { + return await this.workerMutation(MUTATION.refreshSyncJobAuthority, { jobId, leaseToken, expectedGrantRevision: expected.grantRevision, expectedAuthorizationEpoch: expected.authorizationEpoch, expectedCredentialVersion: expected.credentialVersion, expectedScopes: expected.scopes }); + } + + public async restartSyncJobGeneration(jobId: string, leaseToken: string): Promise { + await this.workerMutation(MUTATION.restartSyncJobGeneration, { jobId, leaseToken }); + } + + public async stageSyncAssets(jobId: string, leaseToken: string, index: number, assets: Asset[]): Promise { + await this.workerMutation(MUTATION.stageSyncAssets, { jobId, leaseToken, index, assets }); + } + + public async commitSyncJob(jobId: string, leaseToken: string, chunkCount: number, syncedAt: number, expiresAt: number): Promise { + return await this.workerMutation(MUTATION.commitSyncJob, { jobId, leaseToken, chunkCount, syncedAt, expiresAt }); + } + + public async failSyncJob(jobId: string, leaseToken: string): Promise { + return await this.workerMutation(MUTATION.failSyncJob, { jobId, leaseToken }); + } + + public async deferSyncJob(jobId: string, leaseToken: string, dueAt: number, cooldown?: { context: string; until: number; group?: string }): Promise { + return await this.workerMutation(MUTATION.deferSyncJob, { jobId, leaseToken, dueAt, ...(cooldown === undefined ? {} : { cooldownContext: cooldown.context, cooldownUntil: cooldown.until, ...(cooldown.group === undefined ? {} : { cooldownGroup: cooldown.group }) }) }); + } + + public async cooldown(context: string, group?: string): Promise<{ until: number; group?: string; kind: string } | null> { + return await this.workerQuery<{ until: number; group?: string; kind: string } | null>(QUERY.workerCooldown, { context, ...(group === undefined ? {} : { group }) }); + } + + public async acquireRefreshLease(userId: string, characterId: number, ownerId: string, leaseMs?: number): Promise { + return await this.workerMutation(MUTATION.acquireRefreshLease, { userId, characterId, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) }); + } + + public async renewRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string, leaseMs?: number): Promise { + return await this.workerMutation(MUTATION.renewRefreshLease, { userId, characterId, leaseToken, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) }); + } + + public async releaseRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise { + return await this.workerMutation(MUTATION.releaseRefreshLease, { userId, characterId, leaseToken, ownerId }); + } + + public async markRefreshDispatched(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise { + return await this.workerMutation(MUTATION.markRefreshDispatched, { userId, characterId, leaseToken, ownerId }); + } + + public async commitRefresh(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number, input: { encryptedTokens: string; tokenExpiresAt: number; scopes: string[]; jobId?: string; jobLeaseToken?: string }): Promise { + return await this.workerMutation(MUTATION.commitRefresh, { userId, characterId, leaseToken, ownerId, expectedVersion, ...input }); + } + + public async markRefreshUncertain(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number): Promise { + return await this.workerMutation(MUTATION.markRefreshUncertain, { userId, characterId, leaseToken, ownerId, expectedVersion }); + } + public async beginSync( userId: string, characterId: number, diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 4160575..af0ce7e 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -1,6 +1,7 @@

SHARED DOMAIN VIEW

Assets

All connected characters. The same deterministic report used by the assistant.

{#if failure}{/if} +{#if jobs.length || skipped.length}

Synchronization status

{#each jobs as job}

Character {job.characterId}: {job.status}{job.error ? ` — ${job.error}` : ''}{#if job.status === 'queued' || job.status === 'running'}{/if}

{/each}{#each skipped as entry}

Character {entry.characterId}: {skipLabel(entry.reason)}

{/each}
{/if} diff --git a/tests/infisical.test.ts b/tests/infisical.test.ts index 4c3e027..35464d3 100644 --- a/tests/infisical.test.ts +++ b/tests/infisical.test.ts @@ -20,6 +20,10 @@ describe('Infisical boundary', () => { test('accepts Bun-forwarded commands and preserves child --help arguments', () => { expect(parseArguments(['run', 'docker', 'compose', 'config', '--quiet']).command).toEqual(['docker', 'compose', 'config', '--quiet']); expect(parseArguments(['run', '--deployment', '--', 'bun', 'tool.ts', '--help'])).toMatchObject({ deployment: true, command: ['bun', 'tool.ts', '--help'] }); + expect(parseArguments(['run', '--worker', '--', 'bun', 'workers/sync.ts'])).toMatchObject({ worker: true, command: ['bun', 'workers/sync.ts'] }); + expect(() => parseArguments(['run', '--configure', '--', 'bun', 'scripts/configure-convex.ts'])).toThrow('--configure requires --deployment'); + expect(parseArguments(['run', '--deployment', '--configure', '--', 'bun', 'scripts/configure-convex.ts'])).toMatchObject({ deployment: true, configure: true, command: ['bun', 'scripts/configure-convex.ts'] }); + expect(() => parseArguments(['run', '--worker', '--deployment', '--', 'bun', 'worker.ts'])).toThrow(); expect(parseArguments(['status']).action).toBe('status'); expect(() => parseArguments(['run', '--unknown'])).toThrow(); expect(() => parseArguments(['status', '--', 'printenv'])).toThrow(); @@ -87,10 +91,11 @@ describe('Infisical boundary', () => { test('child processes receive only managed app keys, without bootstrap or ambient admin credentials', () => { const base = { PATH: '/usr/bin', INFISICAL_TOKEN: 'token', [ADMIN_KEY]: 'ambient-admin', ...credentials, TYPESAFE_API_KEY: 'stale-key' }; - const values = { EVE_CLIENT_ID: 'client', EVE_CLIENT_SECRET: 'secret', [ADMIN_KEY]: 'vault-admin', NODE_OPTIONS: 'not-injected' }; + const values = { EVE_CLIENT_ID: 'client', EVE_CLIENT_SECRET: 'secret', CONVEX_BRIDGE_SECRET: 'bridge', CONVEX_SYNC_WORKER_SECRET: 'worker', APP_AUTH_PRIVATE_KEY: 'private', APP_AUTH_JWKS: 'jwks', [ADMIN_KEY]: 'vault-admin', NODE_OPTIONS: 'not-injected' }; const config = { APP_ORIGIN: 'https://dev.evebot.app' }; const env = applicationEnvironment(base, values, config); expect(env.EVE_CLIENT_SECRET).toBe('secret'); + expect(env.CONVEX_SYNC_WORKER_SECRET).toBeUndefined(); expect(env.PATH).toBe('/usr/bin'); expect(env.APP_ORIGIN).toBe(config.APP_ORIGIN); expect(env.TYPESAFE_API_KEY).toBeUndefined(); @@ -98,6 +103,19 @@ describe('Infisical boundary', () => { expect(env.NODE_OPTIONS).toBeUndefined(); expect(Object.keys(env).some(key => key.startsWith('INFISICAL_'))).toBe(false); expect(applicationEnvironment(base, values, config, true)[ADMIN_KEY]).toBe('vault-admin'); + const workerEnv = applicationEnvironment(base, values, config, false, 'worker'); + expect(workerEnv.CONVEX_SYNC_WORKER_SECRET).toBe('worker'); + expect(workerEnv.CONVEX_BRIDGE_SECRET).toBeUndefined(); + expect(workerEnv.APP_AUTH_PRIVATE_KEY).toBeUndefined(); + expect(workerEnv.APP_AUTH_JWKS).toBeUndefined(); + const configureEnv = applicationEnvironment(base, values, { ...config, APP_AUTH_ISSUER: 'https://issuer.example', APP_CAPABILITY_ALLOWLIST: 'asset_sync' }, true, 'configure'); + expect(configureEnv.CONVEX_BRIDGE_SECRET).toBe('bridge'); + expect(configureEnv.CONVEX_SYNC_WORKER_SECRET).toBe('worker'); + expect(configureEnv.APP_AUTH_JWKS).toBe('jwks'); + expect(configureEnv[ADMIN_KEY]).toBe('vault-admin'); + expect(configureEnv.APP_AUTH_ISSUER).toBe('https://issuer.example'); + expect(configureEnv.APP_CAPABILITY_ALLOWLIST).toBe('asset_sync'); + for (const excluded of ['TOKEN_ENCRYPTION_KEY', 'APP_AUTH_PRIVATE_KEY', 'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY']) expect(configureEnv[excluded]).toBeUndefined(); expect(() => applicationEnvironment(base, values, { INFISICAL_TOKEN: 'do-not-forward' })).toThrow('Unexpected public'); expect(() => applicationEnvironment(base, values, { NODE_OPTIONS: 'do-not-execute' })).toThrow('Unexpected public'); }); diff --git a/tests/t07-route-boundaries.test.ts b/tests/t07-route-boundaries.test.ts new file mode 100644 index 0000000..329502d --- /dev/null +++ b/tests/t07-route-boundaries.test.ts @@ -0,0 +1,32 @@ +import { afterEach, expect, test } from 'bun:test'; +import { POST } from '../src/routes/api/sync/cancel/+server'; + +const originalEnv = { APP_MODE: process.env.APP_MODE, APP_ORIGIN: process.env.APP_ORIGIN, APP_CAPABILITY_ALLOWLIST: process.env.APP_CAPABILITY_ALLOWLIST }; + +afterEach(() => { + for (const [key, value] of Object.entries(originalEnv)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } +}); + +function event(request: Request): Parameters[0] { + return { request, url: new URL(request.url), cookies: { get: () => undefined } as any, getClientAddress: () => '127.0.0.1' } as Parameters[0]; +} + +test('sync cancellation rejects hostile origins before session or body work', async () => { + process.env.APP_MODE = 'live'; process.env.APP_ORIGIN = 'http://localhost:5173'; + process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync'; + await expect(POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'https://evil.example', 'content-type': 'application/json' }, body: '{}' })))).rejects.toMatchObject({ status: 403 }); +}); + +test('sync cancellation returns unauthenticated before reading a request body', async () => { + process.env.APP_MODE = 'live'; process.env.APP_ORIGIN = 'http://localhost:5173'; + process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync'; + const response = await POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'http://localhost:5173', 'content-type': 'application/json' }, body: '{}' }))); + expect(response.status).toBe(401); +}); + +test('sync cancellation applies the bounded JSON byte cap', async () => { + process.env.APP_MODE = 'demo'; process.env.APP_ORIGIN = 'http://localhost:5173'; + process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync'; + const response = POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'http://localhost:5173', 'content-type': 'application/json' }, body: JSON.stringify({ jobId: 'x'.repeat(17_000) }) }))); + await expect(response).rejects.toMatchObject({ status: 413 }); +}); diff --git a/tests/t07-sync.test.ts b/tests/t07-sync.test.ts new file mode 100644 index 0000000..7a2ea77 --- /dev/null +++ b/tests/t07-sync.test.ts @@ -0,0 +1,482 @@ +import { describe, expect, test } from 'bun:test'; +import { makeFunctionReference } from 'convex/server'; +import { convexTest } from 'convex-test'; +import schema from '../src/convex/schema'; +import { EsiRateLimitError } from '../src/lib/server/esi'; +import { processSyncJob } from '../workers/sync'; + +process.env.CONVEX_BRIDGE_SECRET = 'test-bridge-secret'; +process.env.CONVEX_SYNC_WORKER_SECRET = 'test-worker-secret'; +process.env.APP_CAPABILITY_ALLOWLIST = 'asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions'; +const modules = { + 'src/convex/functions.ts': () => import('../src/convex/functions'), + 'src/convex/status.ts': () => import('../src/convex/status'), + 'src/convex/_generated/server.ts': async () => ({}), +}; +const mutation = (name: string) => makeFunctionReference<'mutation', any, any>(`functions:${name}`); +const query = (name: string) => makeFunctionReference<'query', any, any>(`functions:${name}`); +const bridge = { bridgeSecret: 'test-bridge-secret' }; +const worker = { workerSecret: 'test-worker-secret' }; + +async function login(t: ReturnType, input: Record = {}) { + return await t.mutation(mutation('bridgeCompleteLogin'), { + ...bridge, characterId: 700, name: 'Queue Pilot', ownerHash: 'queue-owner', scopes: ['esi-assets.read_assets.v1'], + intent: 'initial_login', capability: 'asset_search', requestedScopes: ['esi-assets.read_assets.v1'], approvedExistingScopes: [], + encryptedTokens: 'encrypted-token', tokenExpiresAt: Date.now() + 60_000, sessionHash: 'queue-session', sessionExpiresAt: Date.now() + 60_000, + ...input, + }); +} + +async function enqueue(t: ReturnType, account: any, characterId = 700, sessionHash = 'queue-session') { + return await t.mutation(mutation('bridgeEnqueueAssetSync'), { ...bridge, userId: account.userId, characterId, initiatingSessionHash: sessionHash }); +} + +async function cloneCharacters(t: ReturnType, userId: string, sourceCharacterId: number, characterIds: number[]) { + await t.run(async (ctx: any) => { + const source = await ctx.db.query('characters').withIndex('by_user_character', (q: any) => q.eq('userId', userId).eq('characterId', sourceCharacterId)).first(); + const token = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', userId).eq('characterId', sourceCharacterId)).first(); + const { _id: _characterId, _creationTime: _characterCreated, ...characterFields } = source; + const { _id: _tokenId, _creationTime: _tokenCreated, ...tokenFields } = token; + for (const characterId of characterIds) { + await ctx.db.insert('characters', { ...characterFields, characterId, name: `Clone ${characterId}` }); + await ctx.db.insert('tokenRecords', { ...tokenFields, characterId }); + } + }); +} + +describe('T07 durable asset scheduling', () => { + test('derives authority, isolates tenants, and deduplicates one active job', async () => { + const t = convexTest({ schema, modules }); + const first = await login(t); + const duplicate = await enqueue(t, first); + const same = await enqueue(t, first); + expect(same.id).toBe(duplicate.id); + const other = await login(t, { characterId: 701, sessionHash: 'other-session', ownerHash: 'other-owner' }); + const otherJob = await enqueue(t, other, 701, 'other-session'); + expect(otherJob.id).not.toBe(duplicate.id); + await expect(t.query(query('bridgeSyncJob'), { ...bridge, userId: other.userId, jobId: duplicate.id })).resolves.toBeNull(); + await expect(t.mutation(mutation('bridgeEnqueueAssetSync'), { ...bridge, userId: other.userId, characterId: 700, initiatingSessionHash: 'other-session' })).rejects.toThrow('active'); + }); + + test('claims only user asset jobs, reclaims an expired lease, and keeps unsupported jobs untouched', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 708, sessionHash: 'claim-filter-session' }); + const job = await enqueue(t, account, 708, 'claim-filter-session'); + await t.run(async (ctx: any) => { + await ctx.db.insert('jobs', { userId: account.userId, kind: 'asset_sync', dataset: 'assets', authority: 'system', ownerKind: 'system', status: 'queued', createdAt: Date.now(), updatedAt: Date.now(), dueAt: Date.now(), attempts: 0, maxAttempts: 5, priority: 0 }); + await ctx.db.insert('jobs', { userId: account.userId, kind: 'reference_sync', dataset: 'reference', authority: 'system', ownerKind: 'system', status: 'queued', createdAt: Date.now(), updatedAt: Date.now(), dueAt: Date.now(), attempts: 0, maxAttempts: 5, priority: 0 }); + }); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-filter', leaseMs: 6 * 60_000 }); + expect(claimed.jobId).toBe(job.id); + await t.run(async (ctx: any) => { + const row = await ctx.db.get(claimed.jobId); + await ctx.db.patch(row._id, { leaseExpiresAt: 0 }); + }); + const reclaimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-reclaim', leaseMs: 6 * 60_000 }); + expect(reclaimed.jobId).toBe(job.id); + expect(reclaimed.attempts).toBe(2); + }); + + test('claims, renews beyond the former five-minute lease, and cancellation fences delivery', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t); + const job = await enqueue(t, account); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-a', leaseMs: 6 * 60_000 }); + expect(claimed).toMatchObject({ jobId: job.id, status: 'running', attempts: 1 }); + await t.run(async (ctx: any) => await ctx.db.patch(claimed.jobId, { leaseExpiresAt: Date.now() + 301_000 })); + await expect(t.mutation(mutation('bridgeRenewSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, workerId: 'worker-a', leaseMs: 6 * 60_000 })).resolves.toBe(true); + await expect(t.mutation(mutation('bridgeCancelSyncJob'), { ...bridge, userId: account.userId, jobId: claimed.jobId, initiatingSessionHash: 'queue-session' })).resolves.toMatchObject({ status: 'cancelled' }); + await expect(t.mutation(mutation('bridgeRenewSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, workerId: 'worker-a', leaseMs: 6 * 60_000 })).resolves.toBe(false); + }); + + test('enforces the per-user queue cap and fair claims across users', async () => { + const t = convexTest({ schema, modules }); + const first = await login(t, { characterId: 709, sessionHash: 'cap-session' }); + await cloneCharacters(t, first.userId, 709, [710, 711, 712, 713, 714, 715, 716, 717, 718]); + for (const characterId of [709, 710, 711, 712, 713, 714, 715, 716]) await enqueue(t, first, characterId, 'cap-session'); + await expect(enqueue(t, first, 717, 'cap-session')).rejects.toThrow('queue is full'); + const second = await login(t, { characterId: 719, sessionHash: 'fair-second-session', ownerHash: 'fair-second-owner' }); + await enqueue(t, second, 719, 'fair-second-session'); + const claimedFirst = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-fair-a' }); + const claimedSecond = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-fair-b' }); + expect(claimedFirst.userId).toBe(first.userId); + expect(claimedSecond.userId).toBe(second.userId); + }); + + test('shared provider cooldown blocks a different character until its eligible time', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 720, sessionHash: 'cooldown-session' }); + await cloneCharacters(t, account.userId, 720, [721]); + const first = await enqueue(t, account, 720, 'cooldown-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-a' }); + const until = Date.now() + 30_000; + await t.mutation(mutation('bridgeDeferSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, dueAt: until, cooldownContext: `character:${account.userId}:720`, cooldownUntil: until, cooldownGroup: 'esi:assets' }); + await enqueue(t, account, 721, 'cooldown-session'); + await expect(t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-b' })).resolves.toBeNull(); + await t.run(async (ctx: any) => { + const cooldown = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', 'group:esi:assets')).first(); + if (cooldown) await ctx.db.patch(cooldown._id, { until: 0 }); + const jobs = await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', account.userId).eq('status', 'queued')).collect(); + for (const job of jobs) await ctx.db.patch(job._id, { dueAt: 0 }); + }); + await expect(t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-c' })).resolves.toMatchObject({ userId: account.userId }); + expect(first.status).toBe('queued'); + }); + + test('stages idempotently and fences stale success and failure after credential rotation', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 703, sessionHash: 'publish-session' }); + const queued = await enqueue(t, account, 703, 'publish-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-publish' }); + const generation = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken }); + const row = { itemId: '9007199254740993', characterId: 703, typeId: 34, quantity: 1, locationId: '60003760', locationType: 'station', locationFlag: 'Hangar', isSingleton: true }; + await t.mutation(mutation('bridgeStageSyncAssets'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, index: 0, assets: [row] }); + await t.mutation(mutation('bridgeStageSyncAssets'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, index: 0, assets: [row] }); + await t.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: account.userId, characterId: 703, encryptedTokens: 'rotated', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'], expectedVersion: 1 }); + const rotatedCharacters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + const rotated = rotatedCharacters.find((entry: any) => entry.characterId === 703); + expect(rotated.syncGeneration).toBeUndefined(); + expect(rotated.datasetStatuses.find((entry: any) => entry.dataset === 'assets')).toMatchObject({ status: 'blocked' }); + await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 1, syncedAt: Date.now(), expiresAt: Date.now() + 60_000 })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow(); + const page = await t.query(query('bridgeAssetsPage'), { ...bridge, userId: account.userId, characterId: 703, paginationOpts: { numItems: 10, cursor: null } }); + expect(page.generation).toBeNull(); + expect(generation).toEqual(expect.any(String)); + expect(queued.status).toBe('queued'); + }); + + test('duplicate commit delivery is idempotent and cancellation fences stale success and failure', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 722, sessionHash: 'duplicate-session' }); + await enqueue(t, account, 722, 'duplicate-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-duplicate' }); + await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken }); + await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }); + await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).resolves.toBe(true); + + const second = await login(t, { characterId: 723, sessionHash: 'cancel-fence-session' }); + await enqueue(t, second, 723, 'cancel-fence-session'); + const cancelled = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cancel-fence' }); + await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken }); + await t.mutation(mutation('bridgeCancelSyncJob'), { ...bridge, userId: second.userId, jobId: cancelled.jobId, initiatingSessionHash: 'cancel-fence-session' }); + await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken })).rejects.toThrow(); + }); + + test('authorization quarantine fences both stale success and stale failure', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 727, sessionHash: 'quarantine-session' }); + await enqueue(t, account, 727, 'quarantine-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-quarantine' }); + await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken }); + await t.mutation(mutation('bridgeQuarantineAuthorization'), { ...bridge, userId: account.userId, characterId: 727 }); + await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow(); + }); + + test('worker retries applied-then-lost staging and commit acknowledgements idempotently', async () => { + const events: string[] = []; + let stageCalls = 0; let commitCalls = 0; + const claimed = { id: 'job-lost', jobId: 'job-lost', userId: 'user-lost', characterId: 728, leaseToken: 'lease-lost', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any; + const store = { + claimSyncJob: async () => claimed, + renewSyncJob: async () => { events.push('renew'); return true; }, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-lost', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }), + beginSyncJob: async () => 'generation-lost', + cooldown: async () => null, + stageSyncAssets: async () => { stageCalls += 1; events.push(`stage${stageCalls}`); if (stageCalls === 1) throw new Error('response lost after stage'); }, + commitSyncJob: async () => { commitCalls += 1; events.push(`commit${commitCalls}`); if (commitCalls === 1) throw new Error('response lost after commit'); return true; }, + failSyncJob: async () => { events.push('fail'); return true; }, + deferSyncJob: async () => { events.push('defer'); return true; }, + }; + await processSyncJob({ + store: store as any, + cipher: { open: async () => ({ characterId: 728, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, + sso: {} as any, + esi: { getAssets: async () => ({ assets: [{ itemId: '1', characterId: 728, typeId: 34, quantity: 1, locationId: '2', locationType: 'station', locationFlag: 'Hangar' }], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any, + workerId: 'worker-lost', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {}, + }); + expect(stageCalls).toBe(2); + expect(commitCalls).toBe(2); + expect(events).toEqual(['stage1', 'renew', 'stage2', 'commit1', 'renew', 'commit2']); + }); + + test('ambiguous begin and commit mutations defer for lease recovery instead of failing terminally', async () => { + const beginEvents: string[] = []; + const beginStore = { + claimSyncJob: async () => ({ id: 'job-begin-uncertain', jobId: 'job-begin-uncertain', userId: 'user-begin', characterId: 740, leaseToken: 'lease-begin', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any), + renewSyncJob: async () => { beginEvents.push('renew'); return true; }, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-begin', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }), + beginSyncJob: async () => { beginEvents.push('begin'); throw new Error('response lost after begin'); }, + deferSyncJob: async () => { beginEvents.push('defer'); return true; }, + failSyncJob: async () => { beginEvents.push('fail'); return true; }, + }; + await processSyncJob({ store: beginStore as any, cipher: { open: async () => ({ characterId: 740, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, sso: {} as any, esi: {} as any, workerId: 'worker-begin', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} }); + expect(beginEvents).toEqual(['begin', 'renew', 'begin', 'defer']); + + const commitEvents: string[] = []; + const commitStore = { + claimSyncJob: async () => ({ id: 'job-commit-uncertain', jobId: 'job-commit-uncertain', userId: 'user-commit', characterId: 741, leaseToken: 'lease-commit', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any), + renewSyncJob: async () => { commitEvents.push('renew'); return true; }, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-commit', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }), + beginSyncJob: async () => 'generation-commit-uncertain', + cooldown: async () => null, + stageSyncAssets: async () => { commitEvents.push('stage'); }, + commitSyncJob: async () => { commitEvents.push('commit'); throw new Error('response lost after commit'); }, + deferSyncJob: async () => { commitEvents.push('defer'); return true; }, + failSyncJob: async () => { commitEvents.push('fail'); return true; }, + }; + await processSyncJob({ store: commitStore as any, cipher: { open: async () => ({ characterId: 741, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, sso: {} as any, esi: { getAssets: async () => ({ assets: [], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any, workerId: 'worker-commit', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} }); + expect(commitEvents).toEqual(['commit', 'renew', 'commit', 'defer']); + }); + + test('ambiguous restart after refresh commit defers without replaying the remote refresh', async () => { + const events: string[] = []; + const store = { + claimSyncJob: async () => ({ id: 'job-restart-uncertain', jobId: 'job-restart-uncertain', userId: 'user-restart', characterId: 742, leaseToken: 'lease-restart', status: 'running', attempts: 1, generation: 'old-generation', dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any), + renewSyncJob: async () => { events.push('renew'); return true; }, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-restart', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + markRefreshDispatched: async () => { events.push('dispatch'); return true; }, + commitRefresh: async () => { events.push('refresh-commit'); return { credentialVersion: 2, grantRevision: 1, authorizationEpoch: 1, scopes: ['esi-assets.read_assets.v1'] }; }, + restartSyncJobGeneration: async () => { events.push('restart'); throw new Error('response lost after restart'); }, + markRefreshUncertain: async () => { events.push('uncertain'); return true; }, + deferSyncJob: async () => { events.push('defer'); return true; }, + failSyncJob: async () => { events.push('fail'); return true; }, + }; + await processSyncJob({ store: store as any, cipher: { open: async () => ({ characterId: 742, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'] }), seal: async () => 'rotated-sealed' } as any, sso: { refresh: async () => { events.push('remote'); return { characterId: 742, ownerHash: 'owner', accessToken: 'new-access', refreshToken: 'new-refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }; } } as any, esi: {} as any, workerId: 'worker-restart', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} }); + expect(events).toEqual(['dispatch', 'remote', 'refresh-commit', 'restart', 'renew', 'restart', 'uncertain', 'defer']); + }); + + test('worker publishes a complete empty snapshot through the leased lifecycle', async () => { + const events: string[] = []; + const claimed = { id: 'job-1', jobId: 'job-1', userId: 'user-1', characterId: 700, leaseToken: 'lease-1', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any; + const store = { + claimSyncJob: async () => claimed, + renewSyncJob: async () => true, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-1', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + releaseRefreshLease: async () => { events.push('release'); return true; }, + refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }), + beginSyncJob: async () => { events.push('begin'); return 'generation-1'; }, + cooldown: async () => null, + stageSyncAssets: async () => { events.push('stage'); }, + commitSyncJob: async () => { events.push('commit'); return true; }, + failSyncJob: async () => { events.push('fail'); return true; }, + deferSyncJob: async () => { events.push('defer'); return true; }, + }; + const handled = await processSyncJob({ + store: store as any, + cipher: { open: async () => ({ characterId: 700, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, + sso: { refresh: async () => { throw new Error('refresh should not run'); } } as any, + esi: { getAssets: async () => ({ assets: [], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any, + workerId: 'worker-1', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {}, + }); + expect(handled).toBe(true); + expect(events).toEqual(['release', 'begin', 'commit']); + }); + + test('worker durably marks refresh dispatch before a failed remote attempt', async () => { + const events: string[] = []; + const claimed = { id: 'job-refresh', jobId: 'job-refresh', userId: 'user-refresh', characterId: 725, leaseToken: 'lease-refresh', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any; + const store = { + claimSyncJob: async () => claimed, + renewSyncJob: async () => true, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-dispatch', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + markRefreshDispatched: async () => { events.push('dispatch'); return true; }, + markRefreshUncertain: async () => { events.push('uncertain'); return true; }, + failSyncJob: async () => { events.push('fail'); return true; }, + deferSyncJob: async () => { events.push('defer'); return true; }, + }; + await processSyncJob({ + store: store as any, + cipher: { open: async () => ({ characterId: 725, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, + sso: { refresh: async () => { events.push('remote'); throw new Error('unknown remote outcome'); } } as any, + esi: {} as any, + workerId: 'worker-refresh', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {}, + }); + expect(events).toEqual(['dispatch', 'remote', 'uncertain']); + }); + + test('worker defers rate limits into the durable cooldown instead of failing the job', async () => { + const events: Array<{ name: string; dueAt?: number; cooldown?: unknown }> = []; + const claimed = { id: 'job-2', jobId: 'job-2', userId: 'user-2', characterId: 701, leaseToken: 'lease-2', status: 'running', attempts: 2, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any; + const retryAt = Date.now() + 15_000; + const store = { + claimSyncJob: async () => claimed, + renewSyncJob: async () => true, + acquireRefreshLease: async () => ({ leaseToken: 'refresh-2', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }), + refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }), + beginSyncJob: async () => 'generation-2', + cooldown: async () => null, + stageSyncAssets: async () => {}, + commitSyncJob: async () => true, + failSyncJob: async () => { events.push({ name: 'fail' }); return true; }, + deferSyncJob: async (_jobId: string, _lease: string, dueAt: number, cooldown?: unknown) => { events.push({ name: 'defer', dueAt, cooldown }); return true; }, + }; + await processSyncJob({ + store: store as any, + cipher: { open: async () => ({ characterId: 701, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, + sso: { refresh: async () => { throw new Error('refresh should not run'); } } as any, + esi: { getAssets: async () => { throw new EsiRateLimitError('/latest/characters/701/assets', retryAt, 15_000, 'esi:public'); } } as any, + workerId: 'worker-2', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {}, + }); + expect(events).toHaveLength(1); + expect(events[0]?.name).toBe('defer'); + expect(events[0]?.dueAt).toBe(retryAt); + expect(events[0]?.cooldown).toMatchObject({ context: 'character:user-2:701', until: retryAt }); + }); + + test('terminal retry exhaustion clears syncing state while preserving the prior generation', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 724, sessionHash: 'terminal-session' }); + await enqueue(t, account, 724, 'terminal-session'); + const first = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-terminal-first' }); + await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: first.jobId, leaseToken: first.leaseToken }); + await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: first.jobId, leaseToken: first.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }); + await enqueue(t, account, 724, 'terminal-session'); + const second = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-terminal-second' }); + await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: second.jobId, leaseToken: second.leaseToken }); + await t.run(async (ctx: any) => await ctx.db.patch(second.jobId, { attempts: 5 })); + await expect(t.mutation(mutation('bridgeDeferSyncJob'), { ...worker, jobId: second.jobId, leaseToken: second.leaseToken, dueAt: Date.now() + 1_000 })).resolves.toBe(false); + const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + const character = characters.find((entry: any) => entry.characterId === 724); + expect(character).toMatchObject({ state: 'error', activeGeneration: expect.any(String) }); + expect(character.syncGeneration).toBeUndefined(); + }); + + test('reclaims a lease after refresh commit before generation restart and publishes a fresh generation', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 703, sessionHash: 'refresh-crash-session' }); + const queued = await enqueue(t, account, 703, 'refresh-crash-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-refresh-crash' }); + const oldGeneration = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken }); + const refresh = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash' }); + await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash', leaseToken: refresh.leaseToken }); + await t.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash', leaseToken: refresh.leaseToken, expectedVersion: refresh.expectedVersion, encryptedTokens: 'rotated-token', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken }); + await t.run(async (ctx: any) => await ctx.db.patch(queued.id, { leaseExpiresAt: 0 })); + const reclaimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-refresh-reclaimed' }); + const freshGeneration = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: reclaimed.jobId, leaseToken: reclaimed.leaseToken }); + expect(freshGeneration).not.toBe(oldGeneration); + await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: reclaimed.jobId, leaseToken: reclaimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }); + const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + expect(characters.find((entry: any) => entry.characterId === 703)).toMatchObject({ state: 'ready', activeGeneration: freshGeneration }); + }); + + test('refresh commit validates allowlisted subset scopes and fences reduced grants', async () => { + const makeLease = async (t: ReturnType, account: any, ownerId: string) => { + const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: account.characterId, ownerId }); + await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: account.characterId, ownerId, leaseToken: lease.leaseToken }); + return lease; + }; + const invalid = convexTest({ schema, modules }); + const invalidAccount = await login(invalid, { characterId: 730, sessionHash: 'scope-invalid-session' }); + const invalidLease = await makeLease(invalid, { ...invalidAccount, characterId: 730 }, 'scope-invalid'); + await expect(invalid.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: invalidAccount.userId, characterId: 730, ownerId: 'scope-invalid', leaseToken: invalidLease.leaseToken, expectedVersion: invalidLease.expectedVersion, encryptedTokens: 'bad', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-unknown.scope'] })).rejects.toThrow('Invalid refreshed scopes'); + await expect(invalid.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: invalidAccount.userId, characterId: 730, encryptedTokens: 'expanded', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1', 'esi-wallet.read_character_wallet.v1'], expectedVersion: 1 })).resolves.toBe(false); + const expanded = convexTest({ schema, modules }); + const expandedAccount = await login(expanded, { characterId: 731, sessionHash: 'scope-expanded-session' }); + const expandedLease = await makeLease(expanded, { ...expandedAccount, characterId: 731 }, 'scope-expanded'); + await expect(expanded.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: expandedAccount.userId, characterId: 731, ownerId: 'scope-expanded', leaseToken: expandedLease.leaseToken, expectedVersion: expandedLease.expectedVersion, encryptedTokens: 'bad', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1', 'esi-wallet.read_character_wallet.v1'] })).rejects.toThrow('Invalid refreshed scopes'); + const reduced = convexTest({ schema, modules }); + const reducedAccount = await login(reduced, { characterId: 732, sessionHash: 'scope-reduced-session' }); + const reducedJob = await enqueue(reduced, reducedAccount, 732, 'scope-reduced-session'); + const reducedClaim = await reduced.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'scope-reduced-worker' }); + await reduced.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: reducedClaim.jobId, leaseToken: reducedClaim.leaseToken }); + await reduced.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: reducedClaim.jobId, leaseToken: reducedClaim.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }); + const reducedLease = await makeLease(reduced, { ...reducedAccount, characterId: 732 }, 'scope-reduced'); + await expect(reduced.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: reducedAccount.userId, characterId: 732, ownerId: 'scope-reduced', leaseToken: reducedLease.leaseToken, expectedVersion: reducedLease.expectedVersion, encryptedTokens: 'reduced', tokenExpiresAt: Date.now() + 60_000, scopes: [] })).resolves.toMatchObject({ saved: true }); + const reducedCharacter = (await reduced.query(query('bridgeCharacters'), { ...bridge, userId: reducedAccount.userId })).find((entry: any) => entry.characterId === 732); + expect(reducedCharacter).toMatchObject({ state: 'error' }); + expect(reducedCharacter.activeGeneration).toBeUndefined(); + const unchanged = convexTest({ schema, modules }); + const unchangedAccount = await login(unchanged, { characterId: 733, sessionHash: 'scope-unchanged-session' }); + const unchangedLease = await makeLease(unchanged, { ...unchangedAccount, characterId: 733 }, 'scope-unchanged'); + await expect(unchanged.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: unchangedAccount.userId, characterId: 733, ownerId: 'scope-unchanged', leaseToken: unchangedLease.leaseToken, expectedVersion: unchangedLease.expectedVersion, encryptedTokens: 'unchanged', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'] })).resolves.toMatchObject({ saved: true, grantRevision: 1, credentialVersion: 2 }); + }); + + test('first-generation asset scope loss quarantines staging and clears the matching pointer', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 736, sessionHash: 'first-scope-loss-session' }); + await enqueue(t, account, 736, 'first-scope-loss-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'first-scope-loss-worker' }); + const generation = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken }); + const refresh = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker' }); + await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker', leaseToken: refresh.leaseToken }); + await t.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker', leaseToken: refresh.leaseToken, expectedVersion: refresh.expectedVersion, encryptedTokens: 'reduced-first', tokenExpiresAt: Date.now() + 60_000, scopes: [], jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken }); + const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + const character = characters.find((entry: any) => entry.characterId === 736); + expect(character.state).toBe('error'); + expect(character.syncGeneration).toBeUndefined(); + expect(character.syncLeaseExpiresAt).toBeUndefined(); + expect(character.datasetStatuses.find((entry: any) => entry.dataset === 'assets')).toMatchObject({ status: 'blocked' }); + const job = await t.query(query('bridgeSyncJob'), { ...bridge, userId: account.userId, jobId: claimed.jobId }); + expect(job).toMatchObject({ status: 'quarantined' }); + await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow(); + await t.run(async (ctx: any) => { + const row = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', account.userId).eq('characterId', 736).eq('generation', generation)).first(); + expect(row.status).toBe('failed'); + }); + }); + + test('refresh release is fenced and lets a subsequent worker acquire immediately', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 734, sessionHash: 'release-session' }); + const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner' }); + await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'foreign-owner', leaseToken: lease.leaseToken })).resolves.toBe(false); + await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner', leaseToken: lease.leaseToken })).resolves.toBe(true); + await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner', leaseToken: lease.leaseToken })).resolves.toBe(false); + const next = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner' }); + expect(typeof next.leaseToken).toBe('string'); + expect(next.expectedVersion).toBe(1); + await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner', leaseToken: next.leaseToken }); + await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner', leaseToken: next.leaseToken })).resolves.toBe(false); + }); + + test('authority refresh rejects a same-scope credential rotation instead of adopting newer DB counters', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 704, sessionHash: 'stale-authority-session' }); + const job = await enqueue(t, account, 704, 'stale-authority-session'); + const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-stale' }); + await t.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: account.userId, characterId: 704, encryptedTokens: 'new-token', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'], expectedVersion: 1 }); + await expect(t.mutation(mutation('bridgeRefreshSyncJobAuthority'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, expectedGrantRevision: 1, expectedAuthorizationEpoch: 1, expectedCredentialVersion: 1, expectedScopes: ['esi-assets.read_assets.v1'] })).rejects.toThrow(); + expect(job.status).toBe('queued'); + }); + + test('expired dispatched refresh leases fail closed without replaying a rotated token', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 705, sessionHash: 'dispatch-session' }); + const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-dispatch' }); + await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-dispatch', leaseToken: lease.leaseToken }); + await t.run(async (ctx: any) => { + const row = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', account.userId).eq('characterId', 705)).first(); + await ctx.db.patch(row._id, { expiresAt: 0 }); + }); + await expect(t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-replay' })).resolves.toBeNull(); + const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + expect(characters.find((character: any) => character.characterId === 705)).toMatchObject({ authorizationStatus: 'reconnect_required' }); + await expect(t.query(query('bridgeTokenRecord'), { ...bridge, userId: account.userId, characterId: 705 })).resolves.toBeNull(); + }); + + test('worker credentials cannot invoke general login, enqueue, or cancellation bridges', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 706, sessionHash: 'least-privilege-session' }); + await expect(t.mutation(mutation('bridgeCompleteLogin'), { ...worker, characterId: 707, name: 'Nope', ownerHash: 'nope', scopes: ['esi-assets.read_assets.v1'], intent: 'initial_login', capability: 'asset_search', requestedScopes: ['esi-assets.read_assets.v1'], approvedExistingScopes: [], encryptedTokens: 'x', tokenExpiresAt: Date.now() + 60_000, sessionHash: 'bad-worker-session', sessionExpiresAt: Date.now() + 60_000 })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeEnqueueAssetSync'), { ...worker, userId: account.userId, characterId: 706, initiatingSessionHash: 'least-privilege-session' })).rejects.toThrow(); + await expect(t.mutation(mutation('bridgeCancelSyncJob'), { ...worker, userId: account.userId, jobId: 'missing', initiatingSessionHash: 'least-privilege-session' })).rejects.toThrow(); + }); + + test('refresh lease is durable, CAS committed, and uncertain rotation fails closed', async () => { + const t = convexTest({ schema, modules }); + const account = await login(t, { characterId: 702, sessionHash: 'refresh-session' }); + const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-a' }); + expect(lease).toMatchObject({ expectedVersion: 1 }); + await expect(t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-b' })).resolves.toBeNull(); + await expect(t.mutation(mutation('bridgeMarkRefreshUncertain'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-a', leaseToken: lease.leaseToken, expectedVersion: 1 })).resolves.toBe(true); + const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId }); + expect(characters[0]).toMatchObject({ authorizationStatus: 'reconnect_required' }); + await expect(t.query(query('bridgeTokenRecord'), { ...bridge, userId: account.userId, characterId: 702 })).resolves.toBeNull(); + }); +}); diff --git a/tests/t07-worker-runtime.test.ts b/tests/t07-worker-runtime.test.ts new file mode 100644 index 0000000..a0a8f61 --- /dev/null +++ b/tests/t07-worker-runtime.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from 'bun:test'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { parseSyncEnqueueBody } from '../src/lib/client/sync-queue'; +import { workerEnvironment } from '../scripts/run-sync-worker'; + +describe('T07 worker runtime boundary', () => { + test('bundled worker starts with inert restricted configuration and has no unresolved aliases', async () => { + const directory = await mkdtemp(join(tmpdir(), 'eve-sync-worker-')); + try { + const output = join(directory, 'sync-worker.js'); + const built = await Bun.build({ entrypoints: ['workers/sync.ts'], target: 'bun', outdir: directory, naming: 'sync-worker.js', minify: false }); + expect(built.success).toBe(true); + const source = await readFile(output, 'utf8'); + expect(source.includes('$lib/')).toBe(false); + const env: Record = { PATH: process.env.PATH, HOME: process.env.HOME, CONVEX_URL: 'http://127.0.0.1:9', CONVEX_SYNC_WORKER_SECRET: 'inert-worker-secret', TOKEN_ENCRYPTION_KEY: Buffer.alloc(32).toString('base64'), EVE_CLIENT_ID: 'inert-client', EVE_CALLBACK_URL: 'http://localhost/unused-worker-callback', ESI_USER_AGENT: 'EVE-Intelligence/test', SYNC_WORKER_STARTUP_CHECK: '1' }; + for (const name of ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY']) delete env[name]; + const child = Bun.spawn(['bun', output], { env, stdout: 'pipe', stderr: 'pipe' }); + const code = await child.exited; + const outputText = await new Response(child.stdout).text() + await new Response(child.stderr).text(); + expect(code).toBe(0); + expect(outputText).toBe(''); + } finally { await rm(directory, { recursive: true, force: true }); } + }); + + test('client preserves structured all-skipped and partial enqueue outcomes on HTTP errors', async () => { + expect(parseSyncEnqueueBody({ queued: 0, jobs: [], skipped: [{ characterId: 1, reason: 'queue_full' }], partial: true })).toMatchObject({ queued: 0, skipped: [{ characterId: 1, reason: 'queue_full' }] }); + expect(parseSyncEnqueueBody({ queued: 1, jobs: [{ id: 'job', characterId: 2, status: 'queued', attempts: 0, dueAt: 1, createdAt: 1, updatedAt: 1, leaseExpiresAt: null, cancelRequested: false }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }], partial: true })).toMatchObject({ queued: 1, jobs: [{ id: 'job', characterId: 2 }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }] }); + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = (async () => Response.json({ queued: 0, jobs: [], skipped: [{ characterId: 1, reason: 'queue_full' }], partial: true }, { status: 429 })) as unknown as typeof fetch; + const allSkipped = await (await import('../src/lib/client/sync-queue')).enqueueSync(); + expect(allSkipped.response.status).toBe(429); expect(allSkipped.body).toMatchObject({ jobs: [], skipped: [{ reason: 'queue_full' }] }); + globalThis.fetch = (async () => Response.json({ queued: 1, jobs: [{ id: 'job', characterId: 2, status: 'queued', attempts: 0, dueAt: 1, createdAt: 1, updatedAt: 1, leaseExpiresAt: null, cancelRequested: false }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }], partial: true }, { status: 503 })) as unknown as typeof fetch; + const partial = await (await import('../src/lib/client/sync-queue')).enqueueSync(); + expect(partial.response.status).toBe(503); expect(partial.body).toMatchObject({ jobs: [{ id: 'job' }], skipped: [{ reason: 'enqueue_failed' }] }); + } finally { globalThis.fetch = originalFetch; } + }); + + test('missing status is surfaced and status polling forwards cancellation', async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = (async () => new Response(null, { status: 404 })) as unknown as typeof fetch; + expect(await (await import('../src/lib/client/sync-queue')).syncJobStatus('missing')).toBeNull(); + globalThis.fetch = (() => new Promise(() => {})) as unknown as typeof fetch; + await expect((await import('../src/lib/client/sync-queue')).syncJobStatus('never', undefined, 5)).rejects.toThrow('timed out'); + const controller = new AbortController(); + globalThis.fetch = ((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')), { once: true }); + })) as unknown as typeof fetch; + const pending = (await import('../src/lib/client/sync-queue')).syncJobStatus('running', controller.signal); + controller.abort(); + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }); + } finally { globalThis.fetch = originalFetch; } + }); + + test('worker child environment drops Infisical bootstrap credentials and ambient web keys', () => { + const child = workerEnvironment({ PATH: '/bin', CONVEX_URL: 'http://inert', CONVEX_SYNC_WORKER_SECRET: 'worker-only', TOKEN_ENCRYPTION_KEY: 'inert-key', EVE_CLIENT_ID: 'inert-client', EVE_CLIENT_SECRET: 'inert-eve-secret', INFISICAL_TOKEN: 'inert-token', INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: 'inert-client-id', INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: 'inert-client-secret', CONVEX_BRIDGE_SECRET: 'inert-bridge', APP_AUTH_PRIVATE_KEY: 'inert-jwt', TYPESAFE_API_KEY: 'inert-jev' }); + expect(child).toEqual({ PATH: '/bin', CONVEX_URL: 'http://inert', CONVEX_SYNC_WORKER_SECRET: 'worker-only', TOKEN_ENCRYPTION_KEY: 'inert-key', EVE_CLIENT_ID: 'inert-client', EVE_CLIENT_SECRET: 'inert-eve-secret' }); + expect(Object.keys(child).some((key) => key.startsWith('INFISICAL_'))).toBe(false); + }); + + test('direct worker launcher refuses general bridge/admin/JWT/JEV credentials', async () => { + for (const name of ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY']) { + const forbidden = `inert-${name}`; + const env = { PATH: process.env.PATH, HOME: process.env.HOME, CONVEX_SYNC_WORKER_SECRET: 'inert-worker-secret', [name]: forbidden }; + const child = Bun.spawn(['bun', 'scripts/run-sync-worker.ts'], { env, stdout: 'pipe', stderr: 'pipe' }); + const code = await child.exited; + const output = await new Response(child.stdout).text() + await new Response(child.stderr).text(); + expect(code).not.toBe(0); + expect(output).not.toContain(forbidden); + } + }); +}); diff --git a/tests/workflow-config.test.ts b/tests/workflow-config.test.ts index 95f050f..0a6002b 100644 --- a/tests/workflow-config.test.ts +++ b/tests/workflow-config.test.ts @@ -35,6 +35,8 @@ test('Authenticated route limits remain user-keyed after rollout gates', () => { }); test('Compose and Docker defaults are isolated and digest-pinned', () => { + expect(compose).toMatch(/sync-worker:\n\s+profiles: \["worker"\]/); + expect(compose).toContain('CONVEX_SYNC_WORKER_SECRET: ${CONVEX_SYNC_WORKER_SECRET:-}'); expect(compose).toContain('CONVEX_BIND_ADDRESS:-127.0.0.1'); expect(compose).toContain('WEB_BIND_ADDRESS:-127.0.0.1'); expect(compose).toMatch(/CONVEX_IMAGE:-[^\n]*@sha256:[a-f0-9]{64}/); diff --git a/workers/sync.ts b/workers/sync.ts new file mode 100644 index 0000000..fc8c874 --- /dev/null +++ b/workers/sync.ts @@ -0,0 +1,222 @@ +import { ConvexStore, type RefreshLease } from '../src/lib/server/store'; +import { TokenCipher } from '../src/lib/server/security'; +import { EveSso, type VerifiedGrant } from '../src/lib/server/sso'; +import { EsiClient, EsiError, EsiHttpError, EsiRateLimitError } from '../src/lib/server/esi'; + +/** Conservative prototype budgets; these are intentionally configurable and bounded. */ +export const SYNC_WORKER_DEFAULTS = { + pollMs: 1_000, + leaseMs: 10 * 60_000, + heartbeatMs: 60_000, + maxIdlePolls: 0, +} as const; + +export type WorkerConfig = { + store: ConvexStore; + cipher: TokenCipher; + sso: EveSso; + esi: EsiClient; + workerId: string; + pollMs: number; + leaseMs: number; + heartbeatMs: number; + sleep: (ms: number) => Promise; +}; + +function fixedFailure(): string { return 'Asset synchronization failed'; } +const DEFAULT_SYNC_COOLDOWN_GROUP = 'esi:assets'; +function cooldownContext(userId: string, characterId: number): string { return `character:${userId}:${characterId}`; } + +export class MutationUncertainError extends Error { + constructor(cause: unknown) { super('Synchronization mutation acknowledgement was uncertain'); this.name = 'MutationUncertainError'; this.cause = cause; } +} + +function permanentMutationFailure(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /lease is not valid|authorization (?:is )?stale|authorization changed|generation is not current|already leased|Invalid asset chunk|Asset character mismatch|chunks must be staged|chunk already exists|Snapshot is not staging|Snapshot is incomplete|Invalid snapshot|not active|missing|Invalid refreshed scopes/.test(message); +} + +async function retryMutation(operation: () => Promise, renew?: () => Promise): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 2; attempt += 1) { + try { return await operation(); } + catch (error) { + lastError = error; + if (permanentMutationFailure(error)) throw error; + if (attempt === 0 && renew) { + try { if (!(await renew())) throw new Error('Synchronization lease is not valid'); } + catch (renewError) { + if (permanentMutationFailure(renewError)) throw renewError; + lastError = renewError; + } + } + } + } + throw new MutationUncertainError(lastError); +} + +/** Process one at-least-once delivery. Every write is fenced by the durable lease. */ +export async function processSyncJob(config: WorkerConfig): Promise { + const claimed = await config.store.claimSyncJob(config.workerId, config.leaseMs); + if (!claimed) return false; + let lostLease = false; + let refreshLease: RefreshLease | null = null; + let refreshAttempted = false; + const heartbeat = setInterval(() => { + void config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs).then((ok) => { + if (!ok) lostLease = true; + }).catch(() => { lostLease = true; }); + }, config.heartbeatMs); + const stopHeartbeat = () => clearInterval(heartbeat); + try { + const refresh = await config.store.acquireRefreshLease(claimed.userId, claimed.characterId, config.workerId); + refreshLease = refresh; + if (!refresh) { + await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, Date.now() + 1_000); + return true; + } + + let previous: VerifiedGrant; + try { + previous = await config.cipher.open(refresh.encryptedTokens, `character:${claimed.characterId}`); + if (previous.characterId !== claimed.characterId) throw new Error('authorization mismatch'); + } catch { + await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {}); + return true; + } + + const needsRefresh = refresh.tokenExpiresAt <= Date.now() + 60_000; + let grant: VerifiedGrant = previous; + let authorityVersion = { grantRevision: refresh.grantRevision, authorizationEpoch: refresh.authorizationEpoch, credentialVersion: refresh.credentialVersion, scopes: refresh.scopes }; + if (!needsRefresh) { + // A lease acquired for a still-valid token is only a serialization guard; + // release it before paging so another job is not blocked for two minutes. + await retryMutation(() => config.store.releaseRefreshLease + ? config.store.releaseRefreshLease(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId) + : Promise.resolve(true)); + refreshLease = null; + } + if (needsRefresh) { + try { + const dispatched = await config.store.markRefreshDispatched(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId); + if (!dispatched) { + await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {}); + return true; + } + refreshAttempted = true; + // Exactly one remote refresh attempt. An error is treated as uncertain: + // rotating refresh tokens cannot safely be replayed after a lost result. + grant = await config.sso.refresh(previous.refreshToken); + } catch { + await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {}); + return true; + } + if (grant.characterId !== claimed.characterId || grant.ownerHash !== previous.ownerHash) { + await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {}); + return true; + } + const committed = await config.store.commitRefresh( + claimed.userId, + claimed.characterId, + refresh.leaseToken, + config.workerId, + refresh.expectedVersion, + { encryptedTokens: await config.cipher.seal(grant, `character:${claimed.characterId}`), tokenExpiresAt: grant.expiresAt, scopes: grant.scopes, jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken }, + ); + if (!committed) { + await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {}); + await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {}); + return true; + } + authorityVersion = committed; + // A refreshed credential cannot publish into a generation fenced to the + // old credential version. Discard only that staging generation. + if (claimed.generation !== undefined) await retryMutation( + () => config.store.restartSyncJobGeneration(claimed.jobId, claimed.leaseToken), + () => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs), + ); + } + const currentAuthority = await retryMutation( + () => config.store.refreshSyncJobAuthority(claimed.jobId, claimed.leaseToken, authorityVersion), + () => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs), + ); + if (!grant.scopes.includes('esi-assets.read_assets.v1') || !currentAuthority.scopes.includes('esi-assets.read_assets.v1') || lostLease) { + await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {}); + return true; + } + authorityVersion = currentAuthority; + const generation = await retryMutation( + () => config.store.beginSyncJob(claimed.jobId, claimed.leaseToken), + () => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs), + ); + + const context = cooldownContext(claimed.userId, claimed.characterId); + const cooldown = await config.store.cooldown(context, claimed.cooldownGroup); + if (cooldown && cooldown.until > Date.now()) await config.sleep(cooldown.until - Date.now()); + const authority = { + userId: claimed.userId, + principal: grant.ownerHash, + characterId: claimed.characterId, + scopes: authorityVersion.scopes, + grantRevision: authorityVersion.grantRevision, + authorizationEpoch: authorityVersion.authorizationEpoch, + credentialVersion: authorityVersion.credentialVersion, + } as const; + const snapshot = await config.esi.getAssets(claimed.characterId, grant.accessToken, authority); + const chunks = Math.ceil(snapshot.assets.length / 250); + for (let index = 0; index < chunks; index += 1) { + if (lostLease) throw new Error('lease lost'); + await retryMutation( + () => config.store.stageSyncAssets(claimed.jobId, claimed.leaseToken, index, snapshot.assets.slice(index * 250, (index + 1) * 250)), + () => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs), + ); + } + await retryMutation( + () => config.store.commitSyncJob(claimed.jobId, claimed.leaseToken, chunks, snapshot.syncedAt, snapshot.expiresAt), + () => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs), + ); + return true; + } catch (error) { + if (refreshAttempted && refreshLease) await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refreshLease.leaseToken, config.workerId, refreshLease.expectedVersion).catch(() => {}); + if (error instanceof MutationUncertainError) { + await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, Date.now() + 1_000).catch(() => {}); + } else if (error instanceof EsiRateLimitError) { + const retryAt = Math.min(Date.now() + 24 * 60 * 60_000, Math.max(Date.now() + 1_000, error.retryAt)); + await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, retryAt, { context: cooldownContext(claimed.userId, claimed.characterId), until: retryAt, group: error.cooldown.group ?? DEFAULT_SYNC_COOLDOWN_GROUP }).catch(() => {}); + } else if (error instanceof EsiError && (error.code === 'network' || error.code === 'deadline' || (error instanceof EsiHttpError && (error.status === 408 || error.status === 420 || error.status === 425 || error.status === 429 || error.status >= 500)))) { + const retryAt = Math.min(Date.now() + 24 * 60 * 60_000, Date.now() + 1_000 * 2 ** Math.max(0, claimed.attempts - 1)); + await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, retryAt).catch(() => {}); + } else { + await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {}); + } + return true; + } finally { + stopHeartbeat(); + } +} + +function env(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Sync worker configuration missing: ${name}`); + return value; +} + +export async function runSyncWorker(): Promise { + const workerId = process.env.SYNC_WORKER_ID ?? `sync-worker-${crypto.randomUUID()}`; + const pollMs = Math.max(250, Math.min(30_000, Number(process.env.SYNC_WORKER_POLL_MS ?? SYNC_WORKER_DEFAULTS.pollMs))); + const leaseMs = Math.max(60_000, Math.min(60 * 60_000, Number(process.env.SYNC_WORKER_LEASE_MS ?? SYNC_WORKER_DEFAULTS.leaseMs))); + const heartbeatMs = Math.max(10_000, Math.min(10 * 60_000, Number(process.env.SYNC_WORKER_HEARTBEAT_MS ?? SYNC_WORKER_DEFAULTS.heartbeatMs))); + const store = new ConvexStore(env('CONVEX_URL'), undefined, env('CONVEX_SYNC_WORKER_SECRET')); + const cipher = new TokenCipher(env('TOKEN_ENCRYPTION_KEY')); + const userAgent = env('ESI_USER_AGENT'); + const sso = new EveSso({ clientId: env('EVE_CLIENT_ID'), clientSecret: process.env.EVE_CLIENT_SECRET || undefined, redirectUri: process.env.EVE_CALLBACK_URL ?? 'http://localhost/unused-worker-callback', userAgent }); + const esi = new EsiClient({ userAgent }); + const config: WorkerConfig = { store, cipher, sso, esi, workerId, pollMs, leaseMs, heartbeatMs, sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }; + if (process.env.SYNC_WORKER_STARTUP_CHECK === '1') return; + for (;;) { + const handled = await processSyncJob(config); + if (!handled) await config.sleep(pollMs); + } +} + +if (import.meta.main) await runSyncWorker();