feat: add durable asset synchronization worker

This commit is contained in:
jb
2026-09-19 22:15:23 -04:00
parent 2675ea8114
commit fe7d218195
36 changed files with 2218 additions and 189 deletions
+2
View File
@@ -17,6 +17,8 @@ CONVEX_SELF_HOSTED_URL=http://localhost:3210
CONVEX_SELF_HOSTED_ADMIN_KEY= CONVEX_SELF_HOSTED_ADMIN_KEY=
# Generate independent bridge/encryption/signing values using `bun run keys:generate`. # Generate independent bridge/encryption/signing values using `bun run keys:generate`.
CONVEX_BRIDGE_SECRET= CONVEX_BRIDGE_SECRET=
# Dedicated worker lifecycle credential; never share the bridge secret.
CONVEX_SYNC_WORKER_SECRET=
TOKEN_ENCRYPTION_KEY= TOKEN_ENCRYPTION_KEY=
APP_AUTH_PRIVATE_KEY= APP_AUTH_PRIVATE_KEY=
APP_AUTH_JWKS= APP_AUTH_JWKS=
+2
View File
@@ -5,6 +5,7 @@ COPY package.json bun.lock ./
RUN bun install --frozen-lockfile --ignore-scripts RUN bun install --frozen-lockfile --ignore-scripts
COPY . . COPY . .
RUN bun run build RUN bun run build
RUN bun build workers/sync.ts --target bun --outfile build/sync-worker.js
FROM oven/bun@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS runtime FROM oven/bun@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS runtime
ARG APP_RELEASE_ID=local ARG APP_RELEASE_ID=local
@@ -17,6 +18,7 @@ RUN bun install --frozen-lockfile --production
COPY --from=build /app/build ./build COPY --from=build /app/build ./build
COPY --from=build /app/src/lib ./src/lib COPY --from=build /app/src/lib ./src/lib
COPY --from=build /app/workers ./workers COPY --from=build /app/workers ./workers
COPY --from=build /app/scripts/run-sync-worker.ts ./scripts/run-sync-worker.ts
COPY --from=build /app/fixtures ./fixtures COPY --from=build /app/fixtures ./fixtures
RUN mkdir -p /app/data/reference && chown -R bun:bun /app/data RUN mkdir -p /app/data/reference && chown -R bun:bun /app/data
USER bun USER bun
+8 -4
View File
@@ -44,11 +44,12 @@ Use [the live deployment guide](docs/live-setup.md) for `dev.evebot.app`, the re
bunx convex dev --once bunx convex dev --once
``` ```
`APP_AUTH_ISSUER`, `APP_AUTH_JWKS` and `CONVEX_BRIDGE_SECRET` must be set in **Convex's environment**, not merely the Docker process environment. The configure script handles these three. Custom JWT audience is `eve-intelligence`; the public JWKS data URI avoids a boot dependency on the web server. `APP_AUTH_ISSUER`, `APP_AUTH_JWKS`, `CONVEX_BRIDGE_SECRET` and `CONVEX_SYNC_WORKER_SECRET` must be set in **Convex's environment**, not merely the Docker process environment. The direct configure command reads the local private `.env`, including `CONVEX_SELF_HOSTED_ADMIN_KEY`; never put that admin key in a web or worker environment. Custom JWT audience is `eve-intelligence`; the public JWKS data URI avoids a boot dependency on the web server.
6. Register an EVE application and exact callback at <https://developers.eveonline.com/>. Set `EVE_CLIENT_ID`, `EVE_CALLBACK_URL`, and `EVE_CLIENT_SECRET` for a confidential web client. A public PKCE client leaves the secret blank. Initial login requests `esi-assets.read_assets.v1`; later dataset upgrades preserve only server-allowlisted existing scopes. 6. Register an EVE application and exact callback at <https://developers.eveonline.com/>. Set `EVE_CLIENT_ID`, `EVE_CALLBACK_URL`, and `EVE_CLIENT_SECRET` for a confidential web client. A public PKCE client leaves the secret blank. Initial login requests `esi-assets.read_assets.v1`; later dataset upgrades preserve only server-allowlisted existing scopes.
7. Set `ESI_USER_AGENT` with your actual operator contact. Optional `TYPESAFE_API_KEY` enables server-side JEV routing. Set public/internal Convex URLs as described in `.env.example`. 7. Set `ESI_USER_AGENT` with your actual operator contact. Optional `TYPESAFE_API_KEY` enables server-side JEV routing. Set public/internal Convex URLs as described in `.env.example`.
8. Build the fixture reference database and start `bun run dev`. Connect a character, synchronize it, and add more characters with **Connect EVE Character**. Each connects explicitly to the current application user. 8. Build the fixture reference database and start `bun run dev`. Connect a character, synchronize it, and add more characters with **Connect EVE Character**. Each connects explicitly to the current application user.
9. In a second terminal, create a private `.env.worker` containing only `CONVEX_URL`, `CONVEX_SYNC_WORKER_SECRET`, `TOKEN_ENCRYPTION_KEY`, `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET`, `EVE_CALLBACK_URL`, `ESI_USER_AGENT`, and worker tuning values, then start the worker with `bun --env-file=.env.worker run sync:worker`. Do not source the full web `.env`; the worker launcher intentionally rejects bridge/admin/JWT/JEV credentials.
**Important:** the fixture DB contains only a handful of types. Live assets absent from it remain explicitly unpriced/unknown. Do not describe a fixture-backed report as a complete real-character valuation. See [full-import handoff](docs/reference-operations.md) before ingesting real releases. **Important:** the fixture DB contains only a handful of types. Live assets absent from it remain explicitly unpriced/unknown. Do not describe a fixture-backed report as a complete real-character valuation. See [full-import handoff](docs/reference-operations.md) before ingesting real releases.
@@ -60,10 +61,11 @@ Compose uses Convex's default SQLite backend; it introduces no other datastore.
- `backend`: persistent Convex SQLite volume, API bound to loopback by default (the configured Pangolin deployment explicitly overrides this). - `backend`: persistent Convex SQLite volume, API bound to loopback by default (the configured Pangolin deployment explicitly overrides this).
- `web`: Bun web service with read-only reference volume. - `web`: Bun web service with read-only reference volume.
- `sync-worker`: dedicated durable asset-sync worker with a bundled runtime and worker-only secret.
- `reference-worker`: dedicated one-shot heavy-ingestion process (`tools` profile). - `reference-worker`: dedicated one-shot heavy-ingestion process (`tools` profile).
- `dashboard`: optional loopback-only management UI (`admin` profile). - `dashboard`: optional loopback-only management UI (`admin` profile).
For local container use, set `APP_ORIGIN` and `APP_AUTH_ISSUER` to `http://localhost:3000` and the EVE callback to `http://localhost:3000/auth/eve/callback`. Repeat `convex:configure` after changing issuer. For local container use, set `APP_ORIGIN` and `APP_AUTH_ISSUER` to `http://localhost:3000` and the EVE callback to `http://localhost:3000/auth/eve/callback`. Repeat `bun run convex:configure` after changing issuer.
```sh ```sh
docker compose up -d backend docker compose up -d backend
@@ -71,6 +73,8 @@ docker compose up -d backend
# Explicit production archive import (~109 MiB for the reviewed releases): # Explicit production archive import (~109 MiB for the reviewed releases):
docker compose --profile tools run --rm --build reference-worker docker compose --profile tools run --rm --build reference-worker
docker compose up -d --build web docker compose up -d --build web
# Start the explicitly profiled durable asset worker; Compose forwards only its worker environment:
docker compose --profile worker up -d --build sync-worker
# Optional management UI: # Optional management UI:
docker compose --profile admin up -d dashboard docker compose --profile admin up -d dashboard
``` ```
@@ -109,9 +113,9 @@ docs/research/ current official-source evidence and captured ESI con
## Operational limits ## Operational limits
- The configured development deployment has passed live TypeSafe routing and public Convex WebSocket checks; the operator has confirmed real EVE sign-in, asset synchronization, and matching Assets/chat valuation totals for one character. Live multi-character linking and revocation/transfer scenarios remain separate checks. - This checkpoint records local/disposable prototype verification only. It makes no claim of deployed/live ESI verification or production readiness; live multi-character linking, revocation and transfer scenarios remain deferred.
- Production imports use reviewed source adapters and immutable SQLite artifacts. CCP NPC station IDs, system/type/owner/position facts are imported when `npcStations.jsonl` is present; station names remain optional and the production profile explicitly marks station coverage incomplete until an isolated measured replay. Twenty-one dangling blueprint references in the reviewed release are preserved explicitly rather than fabricated. See reference operations and the verification record for executed import checks and limitations. - Production imports use reviewed source adapters and immutable SQLite artifacts. CCP NPC station IDs, system/type/owner/position facts are imported when `npcStations.jsonl` is present; station names remain optional and the production profile explicitly marks station coverage incomplete until an isolated measured replay. Twenty-one dangling blueprint references in the reviewed release are preserved explicitly rather than fabricated. See reference operations and the verification record for executed import checks and limitations.
- Asset synchronization runs on demand in a single Bun web process. Convex leases/CAS protect staging and token rotation, but durable background character-sync job recovery and scheduled refresh are future work. - Asset synchronization is enqueued by the web process and completed by the separate durable `sync-worker`. Convex leases/CAS protect staging and token rotation; start the worker explicitly with the worker-scoped launcher before testing live synchronization.
- ESI page collections are not transactional snapshots. Stability checks reduce risk; cache rollover/mutation may require a later retry. - ESI page collections are not transactional snapshots. Stability checks reduce risk; cache rollover/mutation may require a later retry.
- Location IDs are shown explicitly as unresolved. Fitted/nested assets are distinct ESI items; no contents multiplier is applied to ship prices. - Location IDs are shown explicitly as unresolved. Fitted/nested assets are distinct ESI items; no contents multiplier is applied to ship prices.
- Conversations/execution summaries persist for authenticated live users; this first UI does not yet provide a history browser. A pending guest request is kept only in tab `sessionStorage` and rechecked after login/sync. - Conversations/execution summaries persist for authenticated live users; this first UI does not yet provide a history browser. A pending guest request is kept only in tab `sessionStorage` and rechecked after login/sync.
+19
View File
@@ -54,6 +54,25 @@ services:
volumes: ["reference-data:/app/data/reference:ro"] volumes: ["reference-data:/app/data/reference:ro"]
depends_on: depends_on:
backend: { condition: service_healthy } backend: { condition: service_healthy }
sync-worker:
profiles: ["worker"]
restart: unless-stopped
build: .
command: ["bun", "scripts/run-sync-worker.ts"]
environment:
CONVEX_URL: http://backend:3210
CONVEX_SYNC_WORKER_SECRET: ${CONVEX_SYNC_WORKER_SECRET:-}
TOKEN_ENCRYPTION_KEY: ${TOKEN_ENCRYPTION_KEY:-}
EVE_CLIENT_ID: ${EVE_CLIENT_ID:-}
EVE_CLIENT_SECRET: ${EVE_CLIENT_SECRET:-}
EVE_CALLBACK_URL: ${EVE_CALLBACK_URL:-http://localhost/unused-worker-callback}
ESI_USER_AGENT: ${ESI_USER_AGENT:-EVE-Intelligence/0.1 (configure operator contact)}
SYNC_WORKER_ID: ${SYNC_WORKER_ID:-local-sync-worker}
SYNC_WORKER_LEASE_MS: ${SYNC_WORKER_LEASE_MS:-600000}
SYNC_WORKER_HEARTBEAT_MS: ${SYNC_WORKER_HEARTBEAT_MS:-60000}
SYNC_WORKER_POLL_MS: ${SYNC_WORKER_POLL_MS:-1000}
depends_on:
backend: { condition: service_healthy }
reference-worker: reference-worker:
profiles: ["tools"] profiles: ["tools"]
build: . build: .
+1 -1
View File
@@ -30,7 +30,7 @@ Dedicated Bun reference worker
EVE authorization-code + PKCE and encrypted token storage runs in Bun. First login creates an app user keyed to a verified character and owner claim; explicit linking attaches additional characters to the current user. Conflicting ownership never silently merges accounts. Session cookies carry random opaque values; only hashes are stored. Application RS256 JWTs are short-lived and audience-scoped for the Convex browser client. EVE access/refresh tokens are never browser credentials. EVE authorization-code + PKCE and encrypted token storage runs in Bun. First login creates an app user keyed to a verified character and owner claim; explicit linking attaches additional characters to the current user. Conflicting ownership never silently merges accounts. Session cookies carry random opaque values; only hashes are stored. Application RS256 JWTs are short-lived and audience-scoped for the Convex browser client. EVE access/refresh tokens are never browser credentials.
Asset sync stages bounded chunks, then atomically changes a character's active generation after all pages succeed. A failed fetch preserves the previous complete generation. Sync metadata records freshness/error, and the domain gate refuses to present missing/stale coverage as complete. ESI page caches are isolated by authorization identity; no partial paginated result is accepted as a full snapshot. Single-host deployment initially runs one web process; token refresh also uses durable compare-and-swap to prevent a stale refresh from overwriting a rotated token. Asset sync is a durable `asset_sync` job owned by the authenticated user; `/api/sync` only enqueues a server-derived, deduplicated request. A dedicated Bun worker claims bounded due work with renewable tokenized leases, stages validated chunks, then atomically changes a character's active generation after all pages succeed. A failed fetch preserves the previous complete generation. Sync metadata records freshness/error, and the domain gate refuses to present missing/stale coverage as complete. ESI page caches are isolated by authorization identity; no partial paginated result is accepted as a full snapshot. Cooldowns are persisted in Convex and shared by workers. Token refresh is coordinated by a durable per-character lease and compare-and-swap; uncertain rotating-token results fail closed as `reconnect_required` instead of replaying a refresh. The worker receives no application signing, JEV, reference, or deployment-admin secrets. This is a local T07 prototype; it requires `CONVEX_SYNC_WORKER_SECRET` and the explicit worker-scoped startup path. T08 location/containment enrichment remains feature-local, with integration at the post-assets/pre-publication seam still pending.
## Valuation ## Valuation
+84
View File
@@ -0,0 +1,84 @@
# T07 durable asset synchronization
Status: prototype implementation. This record describes the local, disposable
queue and does not claim production readiness.
## Boundaries
`POST /api/sync` authenticates the application session and only enqueues
server-derived asset jobs. It does not refresh OAuth credentials or call ESI.
Partial enqueue results carry per-character safe skip reasons; an all-failed
queue response is not reported as `202`. The assets and chat surfaces poll
`/api/sync/status?job=<id>`, expose per-character queued/terminal state and
cancellation, and rerun the report only after every queued job succeeds.
`POST /api/sync/cancel` fences a job for its owning user. The Convex bridge
checks the session proof, character ownership, active grant and exact dataset
scope again when enqueueing and claiming.
Jobs are user-owned `asset_sync` rows. The server derives the dedupe key
`user:character:assets`; only `queued` and `running` rows deduplicate. A claim
captures grant revision, authorization epoch, credential version and scopes.
Claims have random lease tokens and a ten-minute default lease, renewed every
minute by the worker. Expired claims are recovered by the next claim
transaction. Selection is bounded to 100 due rows and prefers users with no
running work before due/creation order, providing per-user fairness without a
second scheduler.
Prototype budgets are conservative and explicit (worker lease/poll values are environment-configurable): five attempts per job,
250 assets per staged chunk, 4,096 chunks, 50,000 ESI rows, at most eight
active jobs per user, a ten-minute sync lease, and a 24-hour maximum retry
delay. ESI request/page/response limits are
owned by the T06 transport. Queue backpressure is visible as a bounded failed
job, never a silently truncated generation.
## Fencing and delivery
A worker starts a generation only through the job lease, stages through a
job-token mutation, and commits success or failure through a mutation that
rechecks the lease, job fence, character authorization and generation fence.
Success and failure therefore cannot mutate state after cancellation, scope
loss, credential rotation, quarantine or lease expiry. Staging is contiguous;
identical duplicate chunk delivery is idempotent while a changed duplicate is
rejected. A successful commit is idempotent if a response is lost. Failed
snapshots preserve the previous complete generation.
`isSingleton` is validated by the T06 ESI parser and is carried unchanged in
every stored asset row; absent legacy values remain unknown.
## Refresh coordination
Refresh leases are durable per user/character rows, not process maps. The
worker decrypts a grant only inside the dedicated worker, performs exactly one
remote rotating-token refresh, then commits the replacement with the expected
credential version. The committed credential version/revision/epoch are the
only authority values used for ESI and publication. A lost, timed-out, or
otherwise uncertain refresh result is never replayed: the character becomes
`reconnect_required`, credentials are removed, active work is quarantined, and
all private datasets are blocked until explicit reconnect.
The worker receives only its dedicated `CONVEX_SYNC_WORKER_SECRET`,
token-encryption, EVE OAuth and ESI user-agent credentials. Worker mutations do
not accept the general `CONVEX_BRIDGE_SECRET`; login/session/tenant operations
remain on the web-only bridge. It does not receive application JWT signing
keys, JEV keys, reference paths/artifacts, or deployment-admin credentials. The
web process no longer performs request-bound refresh or asset paging. The image
build bundles `workers/sync.ts` to `build/sync-worker.js`, so the runtime never
executes a raw `$lib`-aliased TypeScript entry. Compose starts that artifact
through `scripts/run-sync-worker.ts`; the launcher rejects bridge, admin,
JWT-signing and JEV credentials even when `bun run sync:worker` is invoked
directly. Start it with `bun run secrets:run --worker -- docker compose --profile worker up -d
--build sync-worker`, never with the general web/deployment secret modes. The Compose
service is explicitly profiled so plain `docker compose up` cannot launch a worker
with empty credentials.
## Cooldowns and recovery
Provider cooldowns are persisted in `syncCooldowns`, keyed by character
context and shared provider group. Asset jobs carry the provider group, and
claim selection plus the worker both honor the group's eligible-at timestamp
before ESI work. ESI remains the single transport retry/cooldown
implementation. No public-feed scheduler is introduced by this task.
Retention, migrations, production quotas, remote revocation, reference
extraction and public market orchestration remain deferred as required by the
parallel-worktree plan.
+4 -3
View File
@@ -37,7 +37,7 @@ bun run secrets:status
bun run secrets:run -- bun scripts/check-live-config.ts bun run secrets:run -- bun scripts/check-live-config.ts
``` ```
Expected application keys: `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET` for confidential SSO, `TYPESAFE_API_KEY`, `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `APP_AUTH_PRIVATE_KEY`, and `APP_AUTH_JWKS`. Generate the last four **once** using `bun run keys:generate`; import `.env.keys` into the selected Infisical environment without replacing existing API keys. The generator refuses to overwrite an existing file. After verifying the import and key pairing, remove the temporary copy. Back up the vault: changing/loss of the encryption key makes stored EVE credentials unreadable. Do not regenerate keys casually during deployment. Expected application keys: `EVE_CLIENT_ID`, optional `EVE_CLIENT_SECRET` for confidential SSO, `TYPESAFE_API_KEY`, `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `CONVEX_SYNC_WORKER_SECRET`, `APP_AUTH_PRIVATE_KEY`, and `APP_AUTH_JWKS`. Generate the key material **once** using `bun run keys:generate`; import `.env.keys` into the selected Infisical environment without replacing existing API keys. Configure both distinct bridge secrets in Convex with `bun run secrets:run --deployment --configure -- bun run convex:configure`; the sync worker receives only `CONVEX_SYNC_WORKER_SECRET`, while web receives only `CONVEX_BRIDGE_SECRET`. Use `bun run secrets:run --worker -- ...` for worker commands so session/JEV signing and general bridge secrets are not injected. The generator refuses to overwrite an existing file. After verifying the import and key pairing, remove the temporary copy. Back up the vault: changing/loss of the encryption key makes stored EVE credentials unreadable. Do not regenerate keys casually during deployment.
The helper is an operator launcher, not a sandbox. Commands it launches can read injected secrets: never run `env`, `printenv`, shell tracing, or unrestricted `docker compose config` through it. Use `docker compose config --quiet` for validation. Docker administrators can inspect container environments; Infisical startup injection does not conceal secrets from host administrators. Updating a vault value requires recreating the affected container with fresh injected values. Encryption/signing-key rotation needs a separate migration plan. The helper is an operator launcher, not a sandbox. Commands it launches can read injected secrets: never run `env`, `printenv`, shell tracing, or unrestricted `docker compose config` through it. Use `docker compose config --quiet` for validation. Docker administrators can inspect container environments; Infisical startup injection does not conceal secrets from host administrators. Updating a vault value requires recreating the affected container with fresh injected values. Encryption/signing-key rotation needs a separate migration plan.
@@ -47,7 +47,7 @@ Run from the repository root:
```sh ```sh
bun run secrets:run -- docker compose up -d backend bun run secrets:run -- docker compose up -d backend
bun run secrets:run --deployment -- bun run convex:configure bun run secrets:run --deployment --configure -- bun run convex:configure
bun run secrets:run --deployment -- bun node_modules/convex/bin/main.js dev --once --typecheck disable bun run secrets:run --deployment -- bun node_modules/convex/bin/main.js dev --once --typecheck disable
``` ```
@@ -61,9 +61,10 @@ Use the production import procedure in [reference operations](reference-operatio
```sh ```sh
bun run secrets:run -- docker compose up -d --build web bun run secrets:run -- docker compose up -d --build web
bun run secrets:run --worker -- docker compose --profile worker up -d --build sync-worker
``` ```
The container listens on 3000 internally but publishes **5173** on the host. This is the production SvelteKit/Bun server, not the Vite development server. The web mounts reference data read-only. The one-shot reference worker has the writable volume mount. The web container listens on 3000 internally but publishes **5173** on the host. This is the production SvelteKit/Bun server, not the Vite development server. The web mounts reference data read-only. The dedicated sync-worker is a separate long-running service: its image starts the bundled `build/sync-worker.js` through the restricted launcher and receives only worker-safe secrets. The one-shot reference worker has the writable volume mount.
## First live verification ## First live verification
+16 -11
View File
@@ -8,17 +8,20 @@
**Planning baseline:** `fdcc486` (`Initial Commit`). The roadmap records an earlier 97-test gate; each prototype change reruns the current local suite. Existing `jobs`, `userPreferences` and `referenceSyncMetadata` tables are placeholders, not services. **Planning baseline:** `fdcc486` (`Initial Commit`). The roadmap records an earlier 97-test gate; each prototype change reruns the current local suite. Existing `jobs`, `userPreferences` and `referenceSyncMetadata` tables are placeholders, not services.
**T06 evidence:** The centralized ESI client now has fixed typed asset, asset-name, **T06/T07 evidence:** The centralized ESI client has fixed typed asset, asset-name,
asset-location, wallet-balance, and market-read strategies with bounded response asset-location, wallet-balance, and market-read strategies with bounded response
bytes/rows/pages, bounded retries, abort-first cleanup, validators, canonical int64 bytes/rows/pages, bounded retries, abort-first cleanup, validators, canonical int64
helper bodies, complete scoped authority-context cache identity, cooldown queries, helper bodies, complete scoped authority-context cache identity, cooldown queries,
and runtime schemas. Asset snapshots report bounded non-atomic coverage; page-one and runtime schemas. T07 durable scheduling/worker orchestration is implemented as
revalidation is not presented as a transaction. T07 durable scheduling/worker a local prototype: the current full suite records 204 passing tests, including the
orchestration, T08 location/service orchestration, and T13 wallet persistence/UI/ bundled dedicated worker, durable leases, fencing, cancellation/recovery, refresh
service orchestration remain unimplemented; production-readiness gates remain coordination and restricted startup. T08 location/service orchestration and T13
deferred. The narrow `beginSync` expected-authority fence is part of T06's atomic wallet persistence/UI/service orchestration remain pending; T07/T08 integration is
bridge contract: it prevents transport cache authority from being published under not complete and production-readiness gates remain deferred. No deployed/live ESI
a different T05 grant/revision/credential fence; it is not T07 durable worker orchestration. claim is made here. The narrow `beginSync` expected-authority fence is part of T06's
atomic bridge contract: it prevents transport cache authority from being published
under a different T05 grant/revision/credential fence; it is not T07 durable worker
orchestration.
## Execution model ## Execution model
@@ -138,13 +141,15 @@ Implement the prototype identity enforcement needed by the pipeline: revalidate
### T07 — Durable asset queue and worker ### T07 — Durable asset queue and worker
**Own:** proposed `workers/sync.ts`, feature-local job/credential modules, job schema/store operations, `/api/sync` and replacement of request-bound sync orchestration; Compose wiring follows T01. **Status:** implemented local prototype; no deployment, live ESI, or production-readiness claim.
**Own:** `workers/sync.ts`, feature-local job/credential modules, job schema/store operations, `/api/sync` and replacement of request-bound sync orchestration; Compose wiring and restricted operator startup.
**Deliver:** enqueue/status instead of long-running request work; typed deduplicated jobs, bounded attempts/due times, cancellation, fair scheduling/backpressure, shared cooldown/error budgets, renewable leases and fenced success **and failure** publication. Stage/validate assets before active-generation publication and retain validated `isSingleton` facts throughout ingestion. **Deliver:** enqueue/status instead of long-running request work; typed deduplicated jobs, bounded attempts/due times, cancellation, fair scheduling/backpressure, shared cooldown/error budgets, renewable leases and fenced success **and failure** publication. Stage/validate assets before active-generation publication and retain validated `isSingleton` facts throughout ingestion.
Share a per-character refresh lease across web and workers, retaining token CAS. At-least-once delivery is not exactly-once remote refresh: an uncertain rotated result produces actionable reauthorization instead of automatic replay. Keep reference extraction separate and minimize worker secrets. Public feeds must be coordinated rather than independently refreshed per process. Share a per-character refresh lease across web and workers, retaining token CAS. At-least-once delivery is not exactly-once remote refresh: an uncertain rotated result produces actionable reauthorization instead of automatic replay. Keep reference extraction separate and minimize worker secrets. Public feeds must be coordinated rather than independently refreshed per process. The worker requires the distinct `CONVEX_SYNC_WORKER_SECRET`; standalone startup uses a worker-only environment and Compose startup uses the explicit `worker` profile.
**Done:** process kill/restart, long paging beyond the old five-minute lease, duplicate delivery, cancellation, scope loss, stale-worker completion/failure, refresh races and uncertain rotation preserve the newest valid generation. Caps fail visibly. User-supplied jobs cannot gain system authority. Isolated migration/rollback and minimal worker configuration are exercised. **Verified locally:** process kill/restart, long paging beyond the old five-minute lease, duplicate delivery, cancellation, scope loss, stale-worker completion/failure, refresh races, uncertain rotation, restricted runtime startup and configuration boundaries. The full suite currently records 204 passing tests. T08 enrichment integration at the post-assets/pre-publication seam remains pending; production migration/release evidence remains deferred.
### T08 — Private locations, custom names and containment ### T08 — Private locations, custom names and containment
+1 -1
View File
@@ -49,7 +49,7 @@ chmod 700 "$work"
# Do not let an ambient shell, repository .env, or prior Compose project alter # Do not let an ambient shell, repository .env, or prior Compose project alter
# this rehearsal. The explicit env file below contains only non-secret values; # this rehearsal. The explicit env file below contains only non-secret values;
# synthetic key values are loaded separately from the mode-0600 key file. # synthetic key values are loaded separately from the mode-0600 key file.
unset COMPOSE_FILE COMPOSE_ENV_FILES COMPOSE_PROJECT_NAME COMPOSE_PROFILES DOCKER_HOST APP_MODE APP_ORIGIN ORIGIN APP_RELEASE_ID APP_CAPABILITY_ALLOWLIST CONVEX_URL PUBLIC_CONVEX_URL CONVEX_SITE_ORIGIN CONVEX_PORT CONVEX_SITE_PORT WEB_PORT CONVEX_IMAGE CONVEX_DASHBOARD_IMAGE EVE_CALLBACK_URL EVE_CLIENT_ID EVE_CLIENT_SECRET ESI_USER_AGENT TYPESAFE_API_KEY APP_AUTH_ISSUER TOKEN_ENCRYPTION_KEY CONVEX_BRIDGE_SECRET APP_AUTH_PRIVATE_KEY APP_AUTH_JWKS CONVEX_SELF_HOSTED_URL CONVEX_SELF_HOSTED_ADMIN_KEY unset COMPOSE_FILE COMPOSE_ENV_FILES COMPOSE_PROJECT_NAME COMPOSE_PROFILES DOCKER_HOST APP_MODE APP_ORIGIN ORIGIN APP_RELEASE_ID APP_CAPABILITY_ALLOWLIST CONVEX_URL PUBLIC_CONVEX_URL CONVEX_SITE_ORIGIN CONVEX_PORT CONVEX_SITE_PORT WEB_PORT CONVEX_IMAGE CONVEX_DASHBOARD_IMAGE EVE_CALLBACK_URL EVE_CLIENT_ID EVE_CLIENT_SECRET ESI_USER_AGENT TYPESAFE_API_KEY APP_AUTH_ISSUER TOKEN_ENCRYPTION_KEY CONVEX_BRIDGE_SECRET CONVEX_SYNC_WORKER_SECRET APP_AUTH_PRIVATE_KEY APP_AUTH_JWKS CONVEX_SELF_HOSTED_URL CONVEX_SELF_HOSTED_ADMIN_KEY
export CONVEX_BIND_ADDRESS=127.0.0.1 CONVEX_PORT=$((3200 + RANDOM % 500)) CONVEX_SITE_PORT=$((3700 + RANDOM % 500)) export CONVEX_BIND_ADDRESS=127.0.0.1 CONVEX_PORT=$((3200 + RANDOM % 500)) CONVEX_SITE_PORT=$((3700 + RANDOM % 500))
export WEB_BIND_ADDRESS=127.0.0.1 WEB_PORT=$((4200 + RANDOM % 500)) export WEB_BIND_ADDRESS=127.0.0.1 WEB_PORT=$((4200 + RANDOM % 500))
export APP_MODE=live APP_RELEASE_ID=restore-fixture APP_CAPABILITY_ALLOWLIST=asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions export APP_MODE=live APP_RELEASE_ID=restore-fixture APP_CAPABILITY_ALLOWLIST=asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions
+6 -2
View File
@@ -1,7 +1,7 @@
# Implementation roadmap # Implementation roadmap
**Status:** planning proposal, not implementation or a public-launch declaration. **Status:** planning proposal, not implementation or a public-launch declaration.
**Baseline:** the operator has verified real EVE sign-in, asset synchronization, and matching Assets/chat reference valuations. The earlier implementation gate recorded 97 passing tests; local prototype checks are rerun as features land. **Baseline:** prior checkpoints historically recorded live sign-in, asset synchronization and matching Assets/chat reference valuations for one character (see [verification](verification.md)); that history is not new T07 evidence. This T07 checkpoint records a local, disposable prototype only, with no new deployed/live ESI or production-readiness claim. T07 durable synchronization is implemented and verified locally with the current 204-test suite. T08 location/containment enrichment remains a separate feature whose post-assets/pre-publication integration is still pending.
**Evidence:** [architecture](architecture.md), [milestone 1](milestone.md), [verification](verification.md), [ESI research](research/eve.md), [expansion data/permission matrix](research/expansion-data-contracts.md), and [reference operations](reference-operations.md). **Evidence:** [architecture](architecture.md), [milestone 1](milestone.md), [verification](verification.md), [ESI research](research/eve.md), [expansion data/permission matrix](research/expansion-data-contracts.md), and [reference operations](reference-operations.md).
## 1. Decisions confirmed for this roadmap ## 1. Decisions confirmed for this roadmap
@@ -16,6 +16,10 @@
New service/table/path names below are proposed implementation boundaries, not existing APIs. Later phases need focused design and current contract verification before coding; they are not one large implementation task. New service/table/path names below are proposed implementation boundaries, not existing APIs. Later phases need focused design and current contract verification before coding; they are not one large implementation task.
### T07 prototype checkpoint
T07 is implemented as a local prototype, not a deployment claim. It requires the distinct `CONVEX_SYNC_WORKER_SECRET` in Convex and the worker environment, alongside `TOKEN_ENCRYPTION_KEY`, EVE client credentials and `ESI_USER_AGENT`. Start Compose workers only through the explicit `worker` profile; standalone development uses a worker-only `.env.worker`. T08 core modules and their integration at the post-assets/pre-publication seam remain pending.
## 2. What exists versus what still needs building ## 2. What exists versus what still needs building
| Area | Current implementation | Important gap | | Area | Current implementation | Important gap |
@@ -24,7 +28,7 @@ New service/table/path names below are proposed implementation boundaries, not e
| Assets | Complete staged snapshots, tenant gates, exact reference totals | Real search/filtering, paging, containment, custom names, meaningful locations, history | | Assets | Complete staged snapshots, tenant gates, exact reference totals | Real search/filtering, paging, containment, custom names, meaningful locations, history |
| References | 53,001 types; recipes, dogma and system/gate tables; immutable publication | Runtime reader exposes only types/version; station table currently empty; search and domain-specific readers missing | | References | 53,001 types; recipes, dogma and system/gate tables; immutable publication | Runtime reader exposes only types/version; station table currently empty; search and domain-specific readers missing |
| Prices | Global ESI average prices, held by the web process | Durable public feed coordination, regional/private books, history, depth allocation, fees | | Prices | Global ESI average prices, held by the web process | Durable public feed coordination, regional/private books, history, depth allocation, fees |
| Sync | On-demand work inside web requests; generation publication and token CAS | `jobs` is schema-only; no durable dispatcher, crash recovery, shared refresh lease or global worker budgets | | Sync | T07 durable queued jobs, dedicated worker, leases, fenced generation publication, cancellation/recovery and shared refresh coordination | T08 enrichment integration and production release evidence remain pending |
| Finances | Signed integer-cent money helpers | Wallets, journals, transactions, own orders, history/coverage and reconciliation | | Finances | Signed integer-cent money helpers | Wallets, journals, transactions, own orders, history/coverage and reconciliation |
| Industry | Recipe data plus placeholder interfaces/economics fields | Owned blueprint instances, skills/jobs, facility assumptions, formulas and optimizer | | Industry | Recipe data plus placeholder interfaces/economics fields | Owned blueprint instances, skills/jobs, facility assumptions, formulas and optimizer |
| Navigation | Exact light-year distance helper and universe data | Gate-distance service, ship-aware capital graph, dynamic constraints and qualified access evidence | | Navigation | Exact light-year distance helper and universe data | Gate-distance service, ship-aware capital graph, dynamic constraints and qualified access evidence |
+2 -2
View File
@@ -7,10 +7,10 @@ These are requirements for the first implementation, not a claim of a production
- Connecting another character is an authenticated, explicit operation. Callback must preserve the initiating user/session binding. Character ownership conflicts fail closed; there is no implicit merge or transfer. - Connecting another character is an authenticated, explicit operation. Callback must preserve the initiating user/session binding. Character ownership conflicts fail closed; there is no implicit merge or transfer.
- Refresh tokens are envelope encrypted using a server-held encryption key with authenticated associated data. No raw refresh token or client secret may be a browser property, public Convex query field, URL parameter, log or exception message. - Refresh tokens are envelope encrypted using a server-held encryption key with authenticated associated data. No raw refresh token or client secret may be a browser property, public Convex query field, URL parameter, log or exception message.
- Browser sessions use high-entropy opaque identifiers, server-side hashes, HttpOnly/SameSite cookies and bounded expiration. Application JWTs use a dedicated issuer/audience/key and short expiry, not EVE bearer tokens. Queries independently enforce the application identity and ownership. - Browser sessions use high-entropy opaque identifiers, server-side hashes, HttpOnly/SameSite cookies and bounded expiration. Application JWTs use a dedicated issuer/audience/key and short expiry, not EVE bearer tokens. Queries independently enforce the application identity and ownership.
- Server bridge authority is distinct from the deployment administrator credential and browser sessions. All privileged Convex entrypoints require it. Deployment keys are not application runtime defaults. - Server bridge authority is distinct from the deployment administrator credential and browser sessions. Web/session operations require the general bridge secret; worker lifecycle operations require a separate `CONVEX_SYNC_WORKER_SECRET` and expose no login, session, tenant-administration or enqueue/cancel authority. Deployment keys are not application runtime defaults.
- State-changing browser requests require an exact allowed origin. Normal app pages use `Referrer-Policy: same-origin`: global `no-referrer` would make native POST navigations send `Origin: null` and fail CSRF verification. Auth responses retain `no-referrer` so callback grant URLs never become referrers. Null/missing/foreign origins remain rejected. No arbitrary user-provided fetch URLs, SQL identifiers, executable function names or tool invocations. - State-changing browser requests require an exact allowed origin. Normal app pages use `Referrer-Policy: same-origin`: global `no-referrer` would make native POST navigations send `Origin: null` and fail CSRF verification. Auth responses retain `no-referrer` so callback grant URLs never become referrers. Null/missing/foreign origins remain rejected. No arbitrary user-provided fetch URLs, SQL identifiers, executable function names or tool invocations.
- Market/ESI errors are redacted before display. Observability reports route, status, timings and cache/rate-limit signals, never Authorization headers, OAuth codes or secrets. - Market/ESI errors are redacted before display. Observability reports route, status, timings and cache/rate-limit signals, never Authorization headers, OAuth codes or secrets.
- Authenticated ESI caches are keyed by character and authorization context. A partial sync cannot replace a complete snapshot. Token refresh is serialized for a character; stale token rotations cannot overwrite newer credentials. - Authenticated ESI caches are keyed by character and authorization context. A partial sync cannot replace a complete snapshot. Durable jobs use tokenized renewable leases and fenced success/failure publication; duplicate staging is idempotent only for identical payloads. Token refresh is serialized per character with durable CAS; stale token rotations cannot overwrite newer credentials, and uncertain rotation outcomes fail closed as `reconnect_required` rather than replaying.
- Fixture/demo data is explicitly opt-in and visibly marked. Live errors never trigger a silent fallback to demo assets or prices; live reference readers require a production-profile artifact. - Fixture/demo data is explicitly opt-in and visibly marked. Live errors never trigger a silent fallback to demo assets or prices; live reference readers require a production-profile artifact.
- Reference archives are untrusted input. The production importer reads members without extracting arbitrary filesystem paths, rejects unsafe paths and unsupported member types, and enforces compressed/decompressed/member limits, XZ memory limits, archive checksums, and schema validation. Artifact paths/manifests must remain inside the configured reference directory. Cache reuse and fallback verify canonical hashes and the active artifact’s bound provenance; fixture caches cannot satisfy a production import. - Reference archives are untrusted input. The production importer reads members without extracting arbitrary filesystem paths, rejects unsafe paths and unsupported member types, and enforces compressed/decompressed/member limits, XZ memory limits, archive checksums, and schema validation. Artifact paths/manifests must remain inside the configured reference directory. Cache reuse and fallback verify canonical hashes and the active artifact’s bound provenance; fixture caches cannot satisfy a production import.
- Infisical bootstrap credentials stay in an owner-only local file, separate from application keys. The launcher pins the approved instance and selected project/environment/path, suppresses raw credential-command output, and does not pass bootstrap credentials or the vault token to the application. Application secrets are injected only from an allowlist; deployment admin access requires an explicit operator command. Docker/host administrators can still inspect container environment values. - Infisical bootstrap credentials stay in an owner-only local file, separate from application keys. The launcher pins the approved instance and selected project/environment/path, suppresses raw credential-command output, and does not pass bootstrap credentials or the vault token to the application. Application secrets are injected only from an allowlist; deployment admin access requires an explicit operator command. Docker/host administrators can still inspect container environment values.
+82 -22
View File
@@ -1,6 +1,6 @@
# Verification record # Verification record
Initial milestone verified locally on 2026-09-18; live development rollout checked on 2026-09-19 UTC with Bun 1.3.14. Dependency versions are pinned in `package.json`/`bun.lock`. This record retains historical verification facts from earlier checkpoints. The current T07 checkpoint is a local/disposable prototype only: it makes no new deployed, live-ESI, or production-readiness claim. Dependency versions are pinned in `package.json`/`bun.lock`.
## Initial offline/disposable-backend milestone ## Initial offline/disposable-backend milestone
@@ -8,19 +8,91 @@ Initial milestone verified locally on 2026-09-18; live development rollout check
- `bun test`: offline tests exercise domain ownership/freshness, precise integer-cent accumulation, unknown/unpriced/BPC behavior, typed JEV routing and abstention, cryptographic envelopes, PKCE, SSO JWT checks, Convex ownership/session/snapshot rules, ESI cache/pagination/limiter behavior, and reference promotion/rollback/source failure handling. Final run: **66 passed, 0 failed**, across 10 files. - `bun test`: offline tests exercise domain ownership/freshness, precise integer-cent accumulation, unknown/unpriced/BPC behavior, typed JEV routing and abstention, cryptographic envelopes, PKCE, SSO JWT checks, Convex ownership/session/snapshot rules, ESI cache/pagination/limiter behavior, and reference promotion/rollback/source failure handling. Final run: **66 passed, 0 failed**, across 10 files.
- `bun run build`: production Bun server builds. `svelte-adapter-bun` may emit an unresolved `bun:sqlite` warning while leaving that Bun builtin external. This is not a missing npm dependency: the resulting server and container were executed successfully under Bun. - `bun run build`: production Bun server builds. `svelte-adapter-bun` may emit an unresolved `bun:sqlite` warning while leaving that Bun builtin external. This is not a missing npm dependency: the resulting server and container were executed successfully under Bun.
- `scripts/smoke-web.ts`: executed against both the built production server and the final Docker web container in explicit fixture mode. Assistant, Characters and Assets SSR routes succeeded. `/value` and `/assets` returned the same 208,600,000 integer cents (2,086,000.00 ISK), two characters and two excluded/unpriced stacks. Foreign origins and oversized request bodies were rejected. - `scripts/smoke-web.ts`: executed against both the built production server and the final Docker web container in explicit fixture mode. Assistant, Characters and Assets SSR routes succeeded. `/value` and `/assets` returned the same 208,600,000 integer cents (2,086,000.00 ISK), two characters and two excluded/unpriced stacks. Foreign origins and oversized request bodies were rejected.
- Browser-bundle inspection: no references to `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `APP_AUTH_PRIVATE_KEY`, `refresh_token` or `TYPESAFE_API_KEY` in `build/client`. - Browser-bundle inspection: no references to `TOKEN_ENCRYPTION_KEY`, `CONVEX_BRIDGE_SECRET`, `CONVEX_SYNC_WORKER_SECRET`, `APP_AUTH_PRIVATE_KEY`, `refresh_token` or `TYPESAFE_API_KEY` in `build/client`.
- `docker compose config --quiet` and `docker build`: passed. The containerized fixture worker also executed with `--network none`; no archives were fetched. - `docker compose config --quiet` and `docker build`: passed. The containerized fixture worker also executed with `--network none`; no archives were fetched.
- Real self-hosted Convex: deployed functions/schema/auth to an isolated disposable backend and executed `scripts/smoke-convex.ts`. Verified session lookup, asset publication, preservation after failed staging, cross-user isolation, safe status projection, correctly signed wrong-audience JWT rejection, and session revocation. This caught and fixed an incorrect attempt to read reserved `aud` as a Convex identity custom claim; audience enforcement belongs in the backend `applicationID` auth configuration. No live EVE credentials, production volumes, or deployed endpoints were used for this T07 checkpoint. Optional disposable Convex smoke procedures remain documented below but are not evidence of a deployment.
The disposable backend/web containers were stopped and test-only environment files and keys removed after verification. Typecheck and all 66 offline tests were rerun successfully without those files. ## Historical verification from prior checkpoints
Tested backend image: The following evidence was recorded before T07 and is retained as historical context. It is not evidence that this new T07 worker prototype has been deployed or exercised against live ESI.
- The initial disposable Convex smoke deployed functions/schema/auth to an isolated backend and verified session lookup, asset publication, preservation after failed staging, cross-user isolation, safe status projection, correctly signed wrong-audience JWT rejection, and session revocation. It caught and fixed an incorrect attempt to read reserved `aud` as a Convex identity custom claim; audience enforcement belongs in the backend `applicationID` auth configuration.
- The disposable backend/web containers were stopped and test-only environment files and keys removed after that verification. Typecheck and all 66 offline tests were rerun successfully without those files.
- Historical tested backend image:
```text
ghcr.io/get-convex/convex-backend@sha256:3aa2ccd6697a75de53948b1ee531d7c9688fe09ef360748d75df46dfc888f69a
```
### Historical live development rollout — 2026-09-19 UTC
- The integrated gate recorded **97 tests passed, 0 failed, across 16 files**; Svelte/TypeScript check passed with 0 errors/0 warnings, and the Docker web build passed.
- An operator-created Infisical project was connected through a read-only Universal Auth identity. Generated app encryption/signing/bridge keys were imported, compared against the staging file, and locally verified for encryption round-trip and signing/JWKS consistency; the temporary `.env.keys` was removed and secret values were not displayed.
- Persistent self-hosted Convex schema/functions/auth configuration was deployed. Its administrator credential was derived locally for explicit deployment commands, not stored in the web environment.
- Pangolin/UniFi routing was verified: `https://dev-convex.evebot.app/version` returned HTTP 200 and a real WSS upgrade with `Origin: https://dev.evebot.app` passed. Pangolin authentication remained allowed on the main app but was disabled on the Convex API resource by the operator.
- One real TypeSafe request through `JevIntentRouter` returned `asset_valuation`, model `jev-1.13.0`, confidence 1, in 915 ms; this proves connectivity/routing for that sample, not calibration.
- One real public ESI `/markets/prices` request passed using pinned compatibility `2026-08-18`, returning fresh integer-cent estimates for Tritanium and Rifter. These remain global reference estimates, not Jita bids or executable prices.
- Real Reference Data/Hoboleaks/CCP archives totaling **114,291,615 compressed bytes (~109 MiB)** were parsed. The corrected Docker worker compiled them with `--network none`, a 4 GiB memory limit and two CPUs, explicitly targeting `/app/data/reference` in the shared volume. Peak RSS was not measured. The full online download path has fixture-based receipt/validator tests; the real full rebuild used already downloaded archives rather than fetching another copy.
- The web container opened the deployed production artifact, resolved Tritanium/Rifter, matched manifest/table counts, and passed SQLite integrity/foreign-key checks: **53,001 types, 5,082 blueprints, 8,490 systems, 13,978 gates, 21 explicitly recorded unresolved blueprint references**. Deployed version: `b9afed7830756a5e617fec31a2a88c917f07b8f64d2bc719e2176fb628c1e98b`.
- Live anonymous web smoke passed after deployment: SSR routes, no demo fallback, asset/chat authentication gates, session-token protection, and CSRF rejection. A scan of **13 deployed client artifacts plus anonymous SSR output** found no managed secret values; bootstrap tokens and the deployment-admin key were absent from the web environment.
A subsequent operator test exposed `ERR_INVALID_RESPONSE` when submitting Connect Character: the global `no-referrer` response policy made native browser form submissions send `Origin: null`, which SvelteKit rejected before OAuth initiation. An isolated temporary-profile Firefox test reproduced that behavior and confirmed that `same-origin` preserves the native form Origin. App pages now use `same-origin` while auth responses retain `no-referrer`; null/missing/foreign origins remain rejected. The rebuilt deployment passed header and anonymous endpoint checks. EVE discovery and a server-to-Convex read also passed; diagnostics wrote no OAuth/character records. The operator then reached EVE login but reported a generic callback failure. Redacted callback diagnostics identified `scopes_invalid` after token verification; the parser was updated to accept EVE's singleton scope-string representation as well as arrays, with signed-token login/refresh regressions and malformed-claim rejection. The revised image and anonymous smoke passed. The operator subsequently confirmed successful character authorization and asset synchronization, including refresh-grant verification and owner continuity before asset publication, and confirmed that live Assets and chat `/value` returned matching totals without sharing the private amount.
Historical audit regressions covered native-form/CSRF boundaries, shared artifact-directory precedence, live readers rejecting fixture profiles, offline no-network behavior, archive receipts/replay, ZIP/TAR checksums and limits, duplicate records/JSON keys, Hoboleaks metadata, and immutable cache provenance/rollback/reuse. See [live setup](live-setup.md) and [reference operations](reference-operations.md).
## T07 durable worker boundary regressions
- Full local `bun test`: **204 passed, 0 failed** across 27 files.
- `bun test tests/t07-sync.test.ts tests/t07-worker-runtime.test.ts tests/t07-route-boundaries.test.ts`: deterministic queue/fencing/retry regressions plus runtime alias/startup, missing-job/AbortSignal polling, HTTP 429/503 structured outcomes, forbidden worker-environment, hostile-origin, session-ordering and bounded-body checks (31 passed in the focused run).
- `bun build workers/sync.ts --target bun`: produces the same alias-resolved worker artifact copied by the Docker build to `build/sync-worker.js`. A no-cache Docker image build completed, and `docker run --rm --network none` started the image's bundled worker through `scripts/run-sync-worker.ts` with inert Convex/EVE values and `SYNC_WORKER_STARTUP_CHECK=1`; no live keys or network calls were used. `scripts/run-sync-worker.ts` is separately tested to reject general bridge/admin/JWT/JEV credentials without echoing them.
- The operational startup is `bun run secrets:run --worker -- docker compose --profile worker up -d --build sync-worker`; `--deployment`/`--configure` is reserved for Convex administration and configuration, not worker startup. Plain `docker compose up` leaves the worker stopped.
### T07 prototype stage manifest
The T07 stage includes the following files. T08 core files are intentionally excluded:
```text ```text
ghcr.io/get-convex/convex-backend@sha256:3aa2ccd6697a75de53948b1ee531d7c9688fe09ef360748d75df46dfc888f69a .env.example
Dockerfile
README.md
compose.yaml
docs/architecture.md
docs/design-records/t07-durable-sync.md
docs/live-setup.md
docs/parallel-worktrees.md
docs/restore-rehearsal.md
docs/roadmap.md
docs/security.md
docs/verification.md
package.json
scripts/configure-convex.ts
scripts/generate-keys.ts
scripts/lib/infisical.ts
scripts/release-check.ts
scripts/run-sync-worker.ts
scripts/secrets.ts
src/convex/functions.ts
src/convex/schema.ts
src/lib/client/sync-queue.ts
src/lib/domain/model.ts
src/lib/server/application.ts
src/lib/server/store.ts
src/routes/+page.svelte
src/routes/api/sync/+server.ts
src/routes/api/sync/cancel/+server.ts
src/routes/api/sync/status/+server.ts
src/routes/assets/+page.svelte
tests/infisical.test.ts
tests/t07-route-boundaries.test.ts
tests/t07-sync.test.ts
tests/t07-worker-runtime.test.ts
tests/workflow-config.test.ts
workers/sync.ts
``` ```
## Reproducing the real Convex smoke Excluded T08 core files: `src/lib/domain/inventory-location.ts`, `src/lib/server/inventory/`, `tests/fixtures/inventory/`, `tests/inventory-containment.test.ts`, and `tests/inventory-location.test.ts`.
## Optional disposable Convex smoke (not part of this checkpoint)
Use a **disposable** backend, set its `CONVEX_SELF_HOSTED_URL` and admin key, generate application keys and configure/deploy as in the README, then run: Use a **disposable** backend, set its `CONVEX_SELF_HOSTED_URL` and admin key, generate application keys and configure/deploy as in the README, then run:
@@ -30,25 +102,13 @@ SMOKE_ALLOW_WRITE=1 bun scripts/smoke-convex.ts
This writes synthetic users/characters and must not be casually pointed at a production deployment. It needs `CONVEX_BRIDGE_SECRET`, `APP_AUTH_ISSUER` and `APP_AUTH_PRIVATE_KEY` in the calling environment. A wrong-audience test is signed locally; no EVE identity is impersonated and no real EVE account is used. This writes synthetic users/characters and must not be casually pointed at a production deployment. It needs `CONVEX_BRIDGE_SECRET`, `APP_AUTH_ISSUER` and `APP_AUTH_PRIVATE_KEY` in the calling environment. A wrong-audience test is signed locally; no EVE identity is impersonated and no real EVE account is used.
## Live development rollout — 2026-09-19 UTC ## Deferred deployment and live-provider evidence
- Latest integrated gate (including native-form, callback diagnostics, and singleton-scope compatibility): **97 tests passed, 0 failed, across 16 files**; Svelte/TypeScript check passed with 0 errors/0 warnings. Docker web build passed. No deployed endpoint, live ESI request, live EVE credential, or production rollout is claimed for this T07 checkpoint. Existing operational procedures remain reference material only; deployment, migration/rollback, live-provider, browser failure-injection and production-readiness evidence are deferred. See [live setup](live-setup.md) and [reference operations](reference-operations.md).
- Connected the operator-created Infisical project through a read-only Universal Auth identity. Generated app encryption/signing/bridge keys were imported by the operator, compared against the staging file, and locally verified for encryption round-trip and signing/JWKS consistency. The temporary `.env.keys` was then removed. Secret values were not displayed.
- Persistent self-hosted Convex schema/functions/auth configuration deployed. Its administrator credential is derived locally for explicit deployment commands, not stored in the web environment.
- Pangolin/UniFi routing was verified: `https://dev-convex.evebot.app/version` returned HTTP 200 and a real WSS upgrade with `Origin: https://dev.evebot.app` passed. Pangolin authentication remains allowed on the main app but was disabled on the Convex API resource by the operator.
- One real TypeSafe request through `JevIntentRouter` returned `asset_valuation`, model `jev-1.13.0`, confidence 1, in 915 ms. This proves connectivity/routing for that sample, not calibration.
- One real public ESI `/markets/prices` request passed using pinned compatibility `2026-08-18`, returning fresh integer-cent estimates for Tritanium and Rifter. These remain global reference estimates, not Jita bids or executable prices.
- Real Reference Data/Hoboleaks/CCP archives totaling **114,291,615 compressed bytes (~109 MiB)** were parsed. The corrected Docker worker compiled them with `--network none`, a 4 GiB memory limit and two CPUs, explicitly targeting `/app/data/reference` in the shared volume. Peak RSS was not measured. The full online download path has fixture-based receipt/validator tests; the real full rebuild used the already downloaded archives rather than fetching another copy.
- The web container's own read-only repository successfully opened the deployed production artifact, resolved Tritanium/Rifter, matched manifest/table counts, and passed SQLite integrity/foreign-key checks: **53,001 types, 5,082 blueprints, 8,490 systems, 13,978 gates, 21 explicitly recorded unresolved blueprint references**. Deployed version: `b9afed7830756a5e617fec31a2a88c917f07b8f64d2bc719e2176fb628c1e98b`.
- Live anonymous web smoke passed after deployment: SSR routes, no demo fallback, asset/chat authentication gates, session-token protection, and CSRF rejection. A scan of **13 deployed client artifacts plus anonymous SSR output** found no managed secret values; bootstrap tokens and the deployment-admin key were absent from the web environment.
A subsequent operator test exposed `ERR_INVALID_RESPONSE` when submitting Connect Character. The global `no-referrer` response policy made native browser form submissions send `Origin: null`; SvelteKit rejected them before OAuth initiation. An isolated, temporary-profile Firefox test reproduced that behavior and confirmed that `same-origin` preserves the native form Origin. App pages now use `same-origin` while auth responses retain `no-referrer`; null/missing/foreign origins remain rejected. The rebuilt deployment passed header and anonymous endpoint checks. EVE discovery and a server-to-Convex read also passed; diagnostics wrote no OAuth/character records. The operator subsequently reached EVE login but reported a generic failure after callback. Redacted callback-stage diagnostics were added and deployed: only fixed failure categories and allowlisted protocol status/claim names are retained; provider descriptions, JWT payloads, credentials, codes, cookies, and user identifiers are excluded. The next redacted callback diagnostic identified `scopes_invalid` after token verification. The parser now accepts EVE's singleton scope-string representation as well as arrays, with signed-token login/refresh regressions and malformed-claim rejection. The revised image and anonymous smoke passed. After that deployment, the operator confirmed successful character authorization and asset synchronization. Explicit sync exercises refresh-grant verification and owner continuity before asset publication. The operator also confirmed that the live Assets page and chat `/value` return matching totals, without sharing the private amount. This completes the initial live sign-in/sync/reference-valuation verification for one character.
Audit regressions cover native-form header/CSRF boundaries, shared artifact-directory precedence, live readers rejecting fixture profiles, offline no-network behavior, archive receipts/replay, ZIP CRC/length/duplicate records, TAR checksums/end markers, duplicate JSON keys, Hoboleaks file metadata, and immutable cache provenance/rollback/reuse. See [live setup](live-setup.md) and [reference operations](reference-operations.md).
## Not claimed as verified ## Not claimed as verified
- Live revocation/character-transfer scenarios and additional-character linking; successful sign-in/sync for one real character is now operator-confirmed. - Live revocation/character-transfer scenarios, additional-character linking, live ESI behavior and deployed-artifact evidence are not claimed for this prototype.
- Broad TypeSafe inference quality or the proposed 0.80 confidence threshold's calibration on EVE requests. - Broad TypeSafe inference quality or the proposed 0.80 confidence threshold's calibration on EVE requests.
- Compatibility with arbitrary future archive releases, station-data coverage, large-scale memory/performance benchmarks, or production-size market analytics. - Compatibility with arbitrary future archive releases, station-data coverage, large-scale memory/performance benchmarks, or production-size market analytics.
- End-to-end browser automation/visual accessibility audit; the web smoke verifies the built server and endpoints, not all browser interactions. - End-to-end browser automation/visual accessibility audit; the web smoke verifies the built server and endpoints, not all browser interactions.
+1
View File
@@ -13,6 +13,7 @@
"reference:fixture": "bun workers/reference.ts --fixture", "reference:fixture": "bun workers/reference.ts --fixture",
"reference:production": "bun workers/reference.ts --production", "reference:production": "bun workers/reference.ts --production",
"reference:rollback": "bun workers/reference.ts --rollback", "reference:rollback": "bun workers/reference.ts --rollback",
"sync:worker": "bun --no-env-file scripts/run-sync-worker.ts",
"convex:dev": "bunx convex dev", "convex:dev": "bunx convex dev",
"convex:configure": "bun scripts/configure-convex.ts", "convex:configure": "bun scripts/configure-convex.ts",
"keys:generate": "bun scripts/generate-keys.ts", "keys:generate": "bun scripts/generate-keys.ts",
+2 -1
View File
@@ -1,7 +1,8 @@
export {}; export {};
// Run only against your intended self-hosted deployment. Values are never printed. // Run only against your intended self-hosted deployment. Values are never printed.
const requiredNames = ['CONVEX_BRIDGE_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const; const requiredNames = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const;
if (!process.env.CONVEX_SELF_HOSTED_URL || !process.env.CONVEX_SELF_HOSTED_ADMIN_KEY) throw new Error('Set self-hosted URL and admin key before configuring Convex.'); if (!process.env.CONVEX_SELF_HOSTED_URL || !process.env.CONVEX_SELF_HOSTED_ADMIN_KEY) throw new Error('Set self-hosted URL and admin key before configuring Convex.');
if (process.env.CONVEX_SYNC_WORKER_SECRET && process.env.CONVEX_SYNC_WORKER_SECRET === process.env.CONVEX_BRIDGE_SECRET) throw new Error('Worker and general bridge secrets must be distinct.');
for (const name of requiredNames) { for (const name of requiredNames) {
const value = process.env[name]; const value = process.env[name];
if (value === undefined || (name !== 'APP_CAPABILITY_ALLOWLIST' && !value)) { if (value === undefined || (name !== 'APP_CAPABILITY_ALLOWLIST' && !value)) {
+2 -1
View File
@@ -6,8 +6,9 @@ const publicJwk = { ...await exportJWK(publicKey), kid: 'app-v1', alg: 'RS256',
const values = { const values = {
TOKEN_ENCRYPTION_KEY: randomBytes(32).toString('base64'), TOKEN_ENCRYPTION_KEY: randomBytes(32).toString('base64'),
CONVEX_BRIDGE_SECRET: randomBytes(32).toString('base64url'), CONVEX_BRIDGE_SECRET: randomBytes(32).toString('base64url'),
CONVEX_SYNC_WORKER_SECRET: randomBytes(32).toString('base64url'),
APP_AUTH_PRIVATE_KEY: Buffer.from(await exportPKCS8(privateKey)).toString('base64'), APP_AUTH_PRIVATE_KEY: Buffer.from(await exportPKCS8(privateKey)).toString('base64'),
APP_AUTH_JWKS: `data:application/json;base64,${Buffer.from(JSON.stringify({ keys: [publicJwk] })).toString('base64')}` APP_AUTH_JWKS: `data:application/json;base64,${Buffer.from(JSON.stringify({ keys: [publicJwk] })).toString('base64')}`
}; };
await writeFile('.env.keys', Object.entries(values).map(([k, v]) => `${k}='${v}'`).join('\n') + '\n', { flag: 'wx', mode: 0o600 }); await writeFile('.env.keys', Object.entries(values).map(([k, v]) => `${k}='${v}'`).join('\n') + '\n', { flag: 'wx', mode: 0o600 });
console.log('Wrote .env.keys (0600). Import its four values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.'); console.log('Wrote .env.keys (0600). Import its five values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.');
+8 -7
View File
@@ -12,9 +12,11 @@ export interface VaultProject {
export const BOOTSTRAP_KEYS = ['INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET'] as const; export const BOOTSTRAP_KEYS = ['INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET'] as const;
export const APPLICATION_KEYS = [ export const APPLICATION_KEYS = [
'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY', 'TOKEN_ENCRYPTION_KEY', 'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY', 'TOKEN_ENCRYPTION_KEY',
'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS',
] as const; ] as const;
export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const; export const WORKER_APPLICATION_KEYS = ['EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_SYNC_WORKER_SECRET'] as const;
export const CONFIGURE_APPLICATION_KEYS = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_JWKS'] as const;
export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const;
export const ADMIN_KEY = 'CONVEX_SELF_HOSTED_ADMIN_KEY'; export const ADMIN_KEY = 'CONVEX_SELF_HOSTED_ADMIN_KEY';
export type Environment = Record<string, string | undefined>; export type Environment = Record<string, string | undefined>;
export type CliRunner = (args: string[], env: Environment) => Promise<string>; export type CliRunner = (args: string[], env: Environment) => Promise<string>;
@@ -157,7 +159,7 @@ const PUBLIC_KEYS = new Set([
'WEB_BIND_ADDRESS', 'CONVEX_BIND_ADDRESS', 'WEB_PORT', 'CONVEX_PORT', 'CONVEX_SITE_PORT', 'WEB_BIND_ADDRESS', 'CONVEX_BIND_ADDRESS', 'WEB_PORT', 'CONVEX_PORT', 'CONVEX_SITE_PORT',
'CONVEX_IMAGE', 'CONVEX_DASHBOARD_IMAGE', 'RELEASE_ENV', 'CONVEX_IMAGE', 'CONVEX_DASHBOARD_IMAGE', 'RELEASE_ENV',
]); ]);
export function applicationEnvironment(base: Environment, secrets: Record<string, string>, publicConfig: Environment, deployment = false): Environment { export function applicationEnvironment(base: Environment, secrets: Record<string, string>, publicConfig: Environment, deployment = false, target: 'web' | 'worker' | 'configure' = 'web'): Environment {
const env = cleanEnvironment(base); const env = cleanEnvironment(base);
// Public file is the configured deployment, not defaults inherited from Bun's .env.local. // Public file is the configured deployment, not defaults inherited from Bun's .env.local.
for (const [key, value] of Object.entries(publicConfig)) { for (const [key, value] of Object.entries(publicConfig)) {
@@ -165,10 +167,9 @@ export function applicationEnvironment(base: Environment, secrets: Record<string
env[key] = value; env[key] = value;
} }
// Managed app keys must come from the selected vault, never a stale ambient .env. // Managed app keys must come from the selected vault, never a stale ambient .env.
for (const key of APPLICATION_KEYS) { for (const key of APPLICATION_KEYS) delete env[key];
delete env[key]; const allowedKeys = target === 'worker' ? WORKER_APPLICATION_KEYS : target === 'configure' ? CONFIGURE_APPLICATION_KEYS : APPLICATION_KEYS.filter((key) => key !== 'CONVEX_SYNC_WORKER_SECRET');
if (secrets[key] !== undefined) env[key] = secrets[key]; for (const key of allowedKeys) if (secrets[key] !== undefined) env[key] = secrets[key];
}
if (deployment && secrets[ADMIN_KEY]) env[ADMIN_KEY] = secrets[ADMIN_KEY]; if (deployment && secrets[ADMIN_KEY]) env[ADMIN_KEY] = secrets[ADMIN_KEY];
return env; return env;
} }
+1 -1
View File
@@ -65,7 +65,7 @@ function trackedFiles(): string[] {
return result.stdout.split('\n').filter(Boolean); return result.stdout.split('\n').filter(Boolean);
} }
const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token']; const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token'];
function browserBundleFiles(directory = 'build/client'): string[] { function browserBundleFiles(directory = 'build/client'): string[] {
try { statSync(directory); } catch { return []; } try { statSync(directory); } catch { return []; }
const files: string[] = []; const files: string[] = [];
+42
View File
@@ -0,0 +1,42 @@
import { existsSync } from 'node:fs';
import { rm } from 'node:fs/promises';
import { join } from 'node:path';
const forbidden = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY'];
const workerEnvironmentKeys = [
'PATH', 'HOME', 'TMPDIR', 'NODE_ENV', 'LANG', 'LC_ALL', 'TZ', 'NODE_EXTRA_CA_CERTS',
'CONVEX_URL', 'CONVEX_SYNC_WORKER_SECRET', 'TOKEN_ENCRYPTION_KEY',
'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'EVE_CALLBACK_URL', 'ESI_USER_AGENT',
'SYNC_WORKER_ID', 'SYNC_WORKER_LEASE_MS', 'SYNC_WORKER_HEARTBEAT_MS', 'SYNC_WORKER_POLL_MS', 'SYNC_WORKER_STARTUP_CHECK',
] as const;
export function workerEnvironment(base: Record<string, string | undefined>): Record<string, string> {
return Object.fromEntries(workerEnvironmentKeys.flatMap((key) => base[key] === undefined ? [] : [[key, base[key] as string]]));
}
export async function runSyncWorkerLauncher(): Promise<void> {
for (const name of forbidden) {
if (process.env[name]) throw new Error(`Sync worker refuses forbidden runtime credential: ${name}`);
}
if (!process.env.CONVEX_SYNC_WORKER_SECRET) throw new Error('CONVEX_SYNC_WORKER_SECRET is required for the sync worker');
const bundled = join(process.cwd(), 'build', 'sync-worker.js');
let entry = bundled;
let temporary = false;
if (!existsSync(entry)) {
entry = join('/tmp', `eve-sync-worker-${process.pid}.js`);
const result = await Bun.build({ entrypoints: ['workers/sync.ts'], target: 'bun', outdir: '/tmp', naming: `eve-sync-worker-${process.pid}.js`, minify: false });
if (!result.success) throw new Error('Could not bundle the sync worker');
temporary = true;
}
const child = Bun.spawn(['bun', entry], { env: workerEnvironment(process.env), stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
const interrupt = () => child.kill('SIGINT');
const terminate = () => child.kill('SIGTERM');
process.on('SIGINT', interrupt); process.on('SIGTERM', terminate);
try { process.exitCode = await child.exited; }
finally {
process.off('SIGINT', interrupt); process.off('SIGTERM', terminate);
if (temporary) await rm(entry, { force: true });
}
}
if (import.meta.main) await runSyncWorkerLauncher();
+11 -4
View File
@@ -10,7 +10,7 @@ const HELP = `Infisical-backed development deployment (secret values are never p
bun run secrets:run -- <trusted command> [arguments...] bun run secrets:run -- <trusted command> [arguments...]
bun run secrets:run --deployment -- <trusted deployment command> [arguments...] bun run secrets:run --deployment -- <trusted deployment command> [arguments...]
Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env) Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env), --worker to inject only worker-safe application secrets, or --deployment --configure for Convex function configuration.
Project metadata: .infisical.json. Public deployment configuration: config/dev.env. Project metadata: .infisical.json. Public deployment configuration: config/dev.env.
Only allowlisted app secrets are injected. --deployment additionally permits the Convex admin key Only allowlisted app secrets are injected. --deployment additionally permits the Convex admin key
and, when absent from the vault, derives it from the local persistent backend without printing it. and, when absent from the vault, derives it from the local persistent backend without printing it.
@@ -23,22 +23,29 @@ export function parseArguments(args: string[]) {
let command: string[] = []; let command: string[] = [];
let credentialsFile = join(homedir(), '.config/evebot/infisical.env'); let credentialsFile = join(homedir(), '.config/evebot/infisical.env');
let deployment = false; let deployment = false;
let worker = false;
let configure = false;
for (let i = 0; i < rest.length; i++) { for (let i = 0; i < rest.length; i++) {
if (rest[i] === '--') { command = rest.slice(i + 1); break; } if (rest[i] === '--') { command = rest.slice(i + 1); break; }
if (rest[i] === '--credentials-file' && rest[i + 1] && !rest[i + 1].startsWith('--')) credentialsFile = resolve(rest[++i]); if (rest[i] === '--credentials-file' && rest[i + 1] && !rest[i + 1].startsWith('--')) credentialsFile = resolve(rest[++i]);
else if (rest[i] === '--deployment') deployment = true; else if (rest[i] === '--deployment') deployment = true;
else if (rest[i] === '--worker') worker = true;
else if (rest[i] === '--configure') configure = true;
// `bun run` consumes its own -- separator before forwarding script arguments. // `bun run` consumes its own -- separator before forwarding script arguments.
else if (action === 'run' && !rest[i].startsWith('-')) { command = rest.slice(i); break; } else if (action === 'run' && !rest[i].startsWith('-')) { command = rest.slice(i); break; }
else throw new Error('Unknown or incomplete secrets helper option'); else throw new Error('Unknown or incomplete secrets helper option');
} }
if (action === 'run' && !command.length) throw new Error('Supply a trusted command after --'); if (action === 'run' && !command.length) throw new Error('Supply a trusted command after --');
if (action === 'status' && command.length) throw new Error('status does not execute a command'); if (action === 'status' && command.length) throw new Error('status does not execute a command');
return { action, command, credentialsFile, deployment }; if (worker && deployment) throw new Error('Worker secret mode cannot request deployment admin credentials');
if (worker && configure) throw new Error('Worker and configure secret modes are mutually exclusive');
if (configure && !deployment) throw new Error('--configure requires --deployment so the Convex admin credential is available');
return { action, command, credentialsFile, deployment, worker, configure };
} }
async function main(args: string[]): Promise<void> { async function main(args: string[]): Promise<void> {
if (!args.length || args[0] === '--help') { console.log(HELP); return; } if (!args.length || args[0] === '--help') { console.log(HELP); return; }
const { action, command, credentialsFile, deployment } = parseArguments(args); const { action, command, credentialsFile, deployment, worker, configure } = parseArguments(args);
const project = validateProject(JSON.parse(await readFile(new URL('../.infisical.json', import.meta.url), 'utf8'))); const project = validateProject(JSON.parse(await readFile(new URL('../.infisical.json', import.meta.url), 'utf8')));
const credentials = await readCredentials(credentialsFile); const credentials = await readCredentials(credentialsFile);
const secrets = await fetchSecrets(project, credentials); const secrets = await fetchSecrets(project, credentials);
@@ -54,7 +61,7 @@ async function main(args: string[]): Promise<void> {
return; return;
} }
const publicConfig = parseEnv(await readFile(new URL('../config/dev.env', import.meta.url), 'utf8')); const publicConfig = parseEnv(await readFile(new URL('../config/dev.env', import.meta.url), 'utf8'));
const env = applicationEnvironment(process.env, secrets, publicConfig, deployment); const env = applicationEnvironment(process.env, secrets, publicConfig, deployment, worker ? 'worker' : configure ? 'configure' : 'web');
if (deployment && !env[ADMIN_KEY]) env[ADMIN_KEY] = await localDeploymentAdmin(env); if (deployment && !env[ADMIN_KEY]) env[ADMIN_KEY] = await localDeploymentAdmin(env);
const child = Bun.spawn(command, { env, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' }); const child = Bun.spawn(command, { env, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
const interrupt = () => child.kill('SIGINT'); const interrupt = () => child.kill('SIGINT');
+608 -4
View File
@@ -20,6 +20,8 @@ const MAX_CHUNKS = 4_096;
const MAX_ERROR_LENGTH = 160; const MAX_ERROR_LENGTH = 160;
const OAUTH_MAX_LIFETIME_MS = 10 * 60 * 1000; const OAUTH_MAX_LIFETIME_MS = 10 * 60 * 1000;
const SNAPSHOT_MAX_LIFETIME_MS = 60 * 60 * 1000; const SNAPSHOT_MAX_LIFETIME_MS = 60 * 60 * 1000;
const MAX_USER_ACTIVE_SYNC_JOBS = 8;
const SYNC_PROVIDER_GROUP = 'esi:assets';
const DATASET_SCOPE_REQUIREMENTS: Record<string, readonly string[]> = { const DATASET_SCOPE_REQUIREMENTS: Record<string, readonly string[]> = {
assets: ['esi-assets.read_assets.v1'], assets: ['esi-assets.read_assets.v1'],
wallet_balance: ['esi-wallet.read_character_wallet.v1'], wallet_balance: ['esi-wallet.read_character_wallet.v1'],
@@ -33,6 +35,7 @@ const CONSENT_INTENTS = ['initial_login', 'link_character', 'upgrade_dataset', '
const CONSENT_CAPABILITIES = Object.keys(CAPABILITY_DEFINITIONS) as Capability[]; const CONSENT_CAPABILITIES = Object.keys(CAPABILITY_DEFINITIONS) as Capability[];
const bridgeArgs = { bridgeSecret: v.string() }; const bridgeArgs = { bridgeSecret: v.string() };
const workerBridgeArgs = { workerSecret: v.string() };
const characterArgs = { userId: v.string(), characterId: v.number() }; const characterArgs = { userId: v.string(), characterId: v.number() };
const assetPageResultValidator = v.object({ const assetPageResultValidator = v.object({
@@ -57,6 +60,12 @@ function assertBridge(secret: string): void {
} }
} }
function assertSyncWorker(secret: string): void {
const expected = process.env.CONVEX_SYNC_WORKER_SECRET;
if (!expected || expected === process.env.CONVEX_BRIDGE_SECRET || !constantTimeEqual(secret, expected)) throw new Error("Sync worker authentication failed");
assertDeploymentCapability('asset_sync');
}
function assertDeploymentCapability(capability: 'asset_sync'): void { function assertDeploymentCapability(capability: 'asset_sync'): void {
let enabled = false; let enabled = false;
try { try {
@@ -81,6 +90,19 @@ function safeSyncError(_error: string): string {
return "Asset synchronization failed"; return "Asset synchronization failed";
} }
function workerJob(job: any): boolean {
return job?.kind === 'asset_sync' && job?.dataset === 'assets' && job?.authority === 'user' && job?.ownerKind === 'user';
}
function authorityMatches(character: any, job: any): boolean {
return Boolean(character && character.authorizationStatus === 'active'
&& character.scopes.includes('esi-assets.read_assets.v1')
&& character.grantRevision === job.grantRevision
&& character.authorizationEpoch === job.authorizationEpoch
&& character.credentialVersion === job.credentialVersion
&& scopesEqual(character.scopes, job.scopes ?? []));
}
function newIdentifier(prefix: string): string { function newIdentifier(prefix: string): string {
return `${prefix}_${crypto.randomUUID()}`; return `${prefix}_${crypto.randomUUID()}`;
} }
@@ -109,6 +131,11 @@ function validActualScopes(scopes: readonly string[]): boolean {
return new Set(scopes).size === scopes.length && scopes.every((scope) => SERVER_SCOPE_ALLOWLIST.includes(scope)); return new Set(scopes).size === scopes.length && scopes.every((scope) => SERVER_SCOPE_ALLOWLIST.includes(scope));
} }
function scopesSubset(subset: readonly string[], superset: readonly string[]): boolean {
const allowed = new Set(superset);
return subset.every((scope) => allowed.has(scope));
}
function datasetStatusesFor(character: any): any[] { function datasetStatusesFor(character: any): any[] {
return character.datasetStatuses; return character.datasetStatuses;
} }
@@ -138,7 +165,7 @@ function updateDatasetStatuses(
invalidatedAt: timestamp, invalidatedAt: timestamp,
...(dataset === 'assets' ? {} : {}), ...(dataset === 'assets' ? {} : {}),
}); });
if (dataset === 'assets' && (old?.status === 'ready' || old?.generation !== undefined || character.activeGeneration !== undefined)) assetsInvalidated = true; if (dataset === 'assets') assetsInvalidated = true;
continue; continue;
} }
// A newly authorized dataset has no coverage. Existing coverage remains // A newly authorized dataset has no coverage. Existing coverage remains
@@ -458,7 +485,7 @@ export const bridgeCompleteLogin = mutationGeneric({
credentialVersion, credentialVersion,
authorizationEpoch, authorizationEpoch,
datasetStatuses: updated.statuses, datasetStatuses: updated.statuses,
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}), ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}),
}; };
if (character) { if (character) {
await ctx.db.patch(character._id, basePatch); await ctx.db.patch(character._id, basePatch);
@@ -492,6 +519,7 @@ export const bridgeCompleteLogin = mutationGeneric({
version: credentialVersion, version: credentialVersion,
}); });
} }
if (character && (grantChanged || updated.assetsInvalidated || credentialVersion !== character.credentialVersion)) await invalidateCharacterJobs(ctx, resolvedUserId, args.characterId, timestamp);
const sessionId = newIdentifier("session"); const sessionId = newIdentifier("session");
await ctx.db.insert("sessions", { await ctx.db.insert("sessions", {
@@ -546,7 +574,7 @@ export const bridgeSaveTokens = mutationGeneric({
assertBridge(args.bridgeSecret); assertBridge(args.bridgeSecret);
if (!validActualScopes(args.scopes)) return false; if (!validActualScopes(args.scopes)) return false;
const character = await findCharacter(ctx, args.userId, args.characterId); const character = await findCharacter(ctx, args.userId, args.characterId);
if (!character || character.authorizationStatus !== "active") return false; if (!character || character.authorizationStatus !== "active" || !scopesSubset(args.scopes, character.scopes ?? [])) return false;
const record = await ctx.db const record = await ctx.db
.query("tokenRecords") .query("tokenRecords")
.withIndex("by_user_character", (q: any) => .withIndex("by_user_character", (q: any) =>
@@ -568,8 +596,9 @@ export const bridgeSaveTokens = mutationGeneric({
grantRevision: nextRevision, grantRevision: nextRevision,
credentialVersion: nextVersion, credentialVersion: nextVersion,
datasetStatuses: updated.statuses, datasetStatuses: updated.statuses,
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}), ...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}),
}); });
if (updated.assetsInvalidated || nextRevision !== character.grantRevision || nextVersion !== character.credentialVersion) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now());
if (args.returnAuthority) { if (args.returnAuthority) {
return { return {
saved: true, saved: true,
@@ -973,6 +1002,581 @@ export const bridgeFailSync = mutationGeneric({
}, },
}); });
/** Return only fields safe for an authenticated status response. */
function safeSyncJob(job: any): any {
const knownErrors = new Set(['Synchronization cancelled', 'Synchronization scheduled for retry', 'Authorization is no longer active', 'Asset synchronization failed']);
const error = typeof job.lastError === 'string' && knownErrors.has(job.lastError) ? job.lastError : undefined;
return {
id: job._id,
dataset: job.dataset ?? 'assets',
characterId: job.characterId,
status: job.status,
attempts: job.attempts ?? 0,
dueAt: job.dueAt ?? job.createdAt,
createdAt: job.createdAt,
updatedAt: job.updatedAt,
leaseExpiresAt: job.status === 'running' ? (job.leaseExpiresAt ?? null) : null,
cancelRequested: job.cancelRequested === true,
...(error === undefined ? {} : { error }),
};
}
function safeJobError(): string { return 'Asset synchronization failed'; }
function activeJobStatus(status: string): boolean { return status === 'queued' || status === 'running'; }
async function findJob(ctx: any, jobId: string): Promise<any | null> {
return await ctx.db.get(jobId as any);
}
async function markTerminalSyncFailure(ctx: any, job: any, error = safeSyncError('terminal failure')): Promise<void> {
if (job.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error });
}
const character = await findCharacter(ctx, job.userId, job.characterId);
if (!character) return;
const assetWasSyncing = character.syncGeneration === job.generation || character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && (entry.status === 'syncing' || entry.status === 'queued'));
if (!assetWasSyncing) return;
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error }
: entry);
// Never clear activeGeneration/syncedAt/expiresAt: the last complete snapshot
// remains the only readable publication until a replacement succeeds.
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error });
}
async function clearMatchingSyncPointer(ctx: any, character: any, generation: string, error = 'Authorization is no longer active'): Promise<boolean> {
if (!character || character.syncGeneration !== generation) return false;
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error, generation: character.activeGeneration ?? undefined }
: entry);
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error });
return true;
}
async function invalidateCharacterJobs(ctx: any, userId: string, characterId: number, timestamp: number): Promise<void> {
const character = await findCharacter(ctx, userId, characterId);
const jobs = await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', userId)).collect();
let staleSyncState = false;
for (const job of jobs) {
if (workerJob(job) && job.characterId === characterId && activeJobStatus(job.status)) {
if (job.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', userId).eq('characterId', characterId).eq('generation', job.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
}
if (character && job.generation !== undefined && character.syncGeneration === job.generation) staleSyncState = true;
await ctx.db.patch(job._id, {
status: 'quarantined', updatedAt: timestamp, leaseExpiresAt: undefined,
leaseToken: undefined, workerId: undefined, cancelRequested: undefined,
lastError: 'Authorization is no longer active', finishedAt: timestamp,
});
}
}
if (character && staleSyncState && character.syncGeneration !== undefined) await clearMatchingSyncPointer(ctx, character, character.syncGeneration);
}
/** Server-derived enqueue. The caller supplies a session proof, never a job authority. */
export const bridgeEnqueueAssetSync = mutationGeneric({
args: {
...bridgeArgs,
userId: v.string(),
characterId: v.number(),
initiatingSessionHash: v.string(),
},
handler: async (ctx, args) => {
assertBridge(args.bridgeSecret);
assertDeploymentCapability('asset_sync');
const timestamp = now();
const initiating = await findSessionByHash(ctx, args.initiatingSessionHash);
if (!isUsableSession(initiating, timestamp) || initiating.userId !== args.userId) throw new Error('Session is no longer valid');
const character = await findCharacter(ctx, args.userId, args.characterId);
if (!character || character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) {
throw new Error('Asset authorization is not active');
}
const dedupeKey = `${args.userId}:character:${args.characterId}:assets`;
const existing = await ctx.db.query('jobs').withIndex('by_dedupe', (q: any) => q.eq('dedupeKey', dedupeKey)).collect();
const active = existing.find((job: any) => activeJobStatus(job.status));
if (active) return safeSyncJob(active);
const queuedJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job));
const runningJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'running')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job));
if (queuedJobs.length + runningJobs.length >= MAX_USER_ACTIVE_SYNC_JOBS) throw new Error('Synchronization queue is full');
const maxAttempts = 5;
const job = await ctx.db.insert('jobs', {
userId: args.userId,
kind: 'asset_sync',
dataset: 'assets',
status: 'queued',
characterId: args.characterId,
dedupeKey,
createdAt: timestamp,
updatedAt: timestamp,
dueAt: timestamp,
attempts: 0,
maxAttempts,
priority: 0,
ownerKind: 'user',
authority: 'user',
grantRevision: character.grantRevision,
authorizationEpoch: character.authorizationEpoch,
credentialVersion: character.credentialVersion,
scopes: [...character.scopes],
cooldownGroup: SYNC_PROVIDER_GROUP,
cancelRequested: false,
});
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
? { ...entry, status: 'queued', grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion, error: undefined }
: entry);
await ctx.db.patch(character._id, { state: 'queued', datasetStatuses: statuses, error: undefined });
return safeSyncJob({ _id: job, dataset: 'assets', characterId: args.characterId, status: 'queued', attempts: 0, dueAt: timestamp, createdAt: timestamp, updatedAt: timestamp, cancelRequested: false });
},
});
export const bridgeSyncJob = queryGeneric({
args: { ...bridgeArgs, userId: v.string(), jobId: v.string() },
handler: async (ctx, args) => {
assertBridge(args.bridgeSecret);
const job = await findJob(ctx, args.jobId);
if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null;
return safeSyncJob(job);
},
});
/** Cancellation is authenticated against the initiating session and fences a running lease immediately. */
export const bridgeCancelSyncJob = mutationGeneric({
args: { ...bridgeArgs, userId: v.string(), jobId: v.string(), initiatingSessionHash: v.string() },
handler: async (ctx, args) => {
assertBridge(args.bridgeSecret);
assertDeploymentCapability('asset_sync');
const initiating = await findSessionByHash(ctx, args.initiatingSessionHash);
if (!isUsableSession(initiating) || initiating.userId !== args.userId) throw new Error('Session is no longer valid');
const job = await findJob(ctx, args.jobId);
if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null;
if (!activeJobStatus(job.status)) return safeSyncJob(job);
const timestamp = now();
const character = job.characterId === undefined ? null : await findCharacter(ctx, args.userId, job.characterId);
if (job.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', args.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Synchronization cancelled' });
}
await ctx.db.patch(job._id, {
status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp,
leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined,
cancelRequested: true, lastError: 'Synchronization cancelled',
});
if (character && character.syncGeneration === job.generation) {
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' }
: entry);
await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Synchronization cancelled' });
} else if (character && character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && entry.status === 'queued')) {
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' } : entry);
await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, error: 'Synchronization cancelled' });
}
return safeSyncJob({ ...job, status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, cancelRequested: true, lastError: 'Synchronization cancelled' });
},
});
/**
* Claim and recover in one transaction. Expired leases are requeued before a
* candidate is selected, making process termination recoverable without a
* second scheduler or an in-memory queue.
*/
export const bridgeClaimSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, workerId: v.string(), leaseMs: v.optional(v.number()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
if (!args.workerId || args.workerId.length > 160) throw new Error('Invalid worker identity');
const timestamp = now();
const leaseMs = args.leaseMs ?? 10 * 60 * 1000;
if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) throw new Error('Invalid sync lease duration');
const expired = await ctx.db.query('jobs').withIndex('by_lease_expires_at', (q: any) => q.lte('leaseExpiresAt', timestamp)).take(100);
for (const job of expired) {
if (!workerJob(job) || job.status !== 'running' || (job.leaseExpiresAt ?? Infinity) > timestamp) continue;
if (job.cancelRequested === true) {
await ctx.db.patch(job._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization cancelled' });
} else if ((job.attempts ?? 0) >= (job.maxAttempts ?? 5)) {
const character = await findCharacter(ctx, job.userId, job.characterId);
if (!authorityMatches(character, job)) {
if (job.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
await clearMatchingSyncPointer(ctx, character, job.generation);
}
await ctx.db.patch(job._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Authorization is no longer active' });
continue;
}
await markTerminalSyncFailure(ctx, job);
await ctx.db.patch(job._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
} else {
await ctx.db.patch(job._id, { status: 'queued', updatedAt: timestamp, dueAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined });
}
}
const candidates = await ctx.db.query('jobs').withIndex('by_asset_sync_queue', (q: any) => q.eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued').lte('dueAt', timestamp)).order('asc').take(100);
if (candidates.length === 0) return null;
// Bounded fair selection: an account with an in-flight job loses to an
// account with no in-flight work; due/creation order breaks ties.
const supportedCandidates = candidates.filter((candidate: any) => workerJob(candidate));
if (supportedCandidates.length === 0) return null;
const runningByUser = new Map<string, number>();
for (const candidate of supportedCandidates) {
if (runningByUser.has(candidate.userId)) continue;
const running = (await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', candidate.userId).eq('status', 'running')).take(20)).filter((job: any) => workerJob(job));
runningByUser.set(candidate.userId, running.length);
}
const ordered = [...supportedCandidates].sort((left: any, right: any) =>
(runningByUser.get(left.userId)! - runningByUser.get(right.userId)!)
|| ((left.priority ?? 0) - (right.priority ?? 0))
|| ((left.dueAt ?? left.createdAt) - (right.dueAt ?? right.createdAt))
|| ((left.lastClaimedAt ?? 0) - (right.lastClaimedAt ?? 0))
|| String(left._id).localeCompare(String(right._id)));
for (const candidate of ordered) {
if (candidate.cancelRequested === true) {
await ctx.db.patch(candidate._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Synchronization cancelled' });
continue;
}
if (candidate.cooldownGroup) {
const shared = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${candidate.cooldownGroup}`)).first();
if (shared && shared.until > timestamp) continue;
}
const character = candidate.characterId === undefined ? null : await findCharacter(ctx, candidate.userId, candidate.characterId);
const authorityCurrent = authorityMatches(character, candidate);
if (!authorityCurrent) {
if (candidate.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', candidate.userId).eq('characterId', candidate.characterId).eq('generation', candidate.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
await clearMatchingSyncPointer(ctx, character, candidate.generation);
}
await ctx.db.patch(candidate._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Authorization is no longer active' });
continue;
}
const leaseToken = newIdentifier('lease');
const attempts = (candidate.attempts ?? 0) + 1;
await ctx.db.patch(candidate._id, { status: 'running', attempts, updatedAt: timestamp, startedAt: candidate.startedAt ?? timestamp, dueAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken, workerId: args.workerId, lastClaimedAt: timestamp });
return { ...safeSyncJob({ ...candidate, status: 'running', attempts, updatedAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken }), jobId: candidate._id, leaseToken, userId: candidate.userId, characterId: candidate.characterId, grantRevision: candidate.grantRevision, authorizationEpoch: candidate.authorizationEpoch, credentialVersion: candidate.credentialVersion, scopes: candidate.scopes ?? [], ...(candidate.generation === undefined ? {} : { generation: candidate.generation }), cooldownGroup: candidate.cooldownGroup ?? SYNC_PROVIDER_GROUP };
}
return null;
},
});
export const bridgeRenewSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), workerId: v.string(), leaseMs: v.optional(v.number()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const job = await findJob(ctx, args.jobId);
if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== args.leaseToken || job.workerId !== args.workerId) return false;
if (job.cancelRequested === true || (job.leaseExpiresAt ?? 0) <= now()) return false;
const leaseMs = args.leaseMs ?? 10 * 60 * 1000;
if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) return false;
await ctx.db.patch(job._id, { leaseExpiresAt: now() + leaseMs, updatedAt: now() });
return true;
},
});
async function jobLease(ctx: any, jobId: string, leaseToken: string): Promise<any> {
const job = await findJob(ctx, jobId);
if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== leaseToken || (job.leaseExpiresAt ?? 0) <= now()) throw new Error('Synchronization lease is not valid');
return job;
}
async function jobCharacter(ctx: any, job: any): Promise<any> {
const character = await findCharacter(ctx, job.userId, job.characterId);
if (!authorityMatches(character, job)) {
throw new Error('Synchronization authorization is stale');
}
return character;
}
export const bridgeRefreshSyncJobAuthority = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), expectedGrantRevision: v.number(), expectedAuthorizationEpoch: v.number(), expectedCredentialVersion: v.number(), expectedScopes: v.array(v.string()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const job = await jobLease(ctx, args.jobId, args.leaseToken);
const character = await findCharacter(ctx, job.userId, job.characterId);
const expected = { grantRevision: args.expectedGrantRevision, authorizationEpoch: args.expectedAuthorizationEpoch, credentialVersion: args.expectedCredentialVersion, scopes: args.expectedScopes };
if (!authorityMatches(character, expected)) throw new Error('Synchronization authorization changed');
if (!authorityMatches(character, job)) throw new Error('Synchronization job authority is stale');
return expected;
},
});
export const bridgeRestartSyncJobGeneration = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const job = await jobLease(ctx, args.jobId, args.leaseToken);
if (job.generation === undefined) return null;
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization refreshed before publication' });
const character = await findCharacter(ctx, job.userId, job.characterId);
if (character?.syncGeneration === job.generation) await ctx.db.patch(character._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined, state: 'queued' });
await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() });
return null;
},
});
export const bridgeBeginSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const job = await jobLease(ctx, args.jobId, args.leaseToken);
let character = await jobCharacter(ctx, job);
if (job.generation !== undefined) {
const existing = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (existing && existing.status === 'staging' && existing.grantRevision === job.grantRevision && existing.authorizationEpoch === job.authorizationEpoch && existing.credentialVersion === job.credentialVersion) return job.generation;
if (existing?.status === 'staging') await ctx.db.patch(existing._id, { status: 'failed', error: 'Synchronization authority changed' });
await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() });
const oldCharacter = await findCharacter(ctx, job.userId, job.characterId);
if (oldCharacter?.syncGeneration === job.generation) await ctx.db.patch(oldCharacter._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined });
// The character read above predates the atomic cleanup. Re-read it before
// checking the lease, otherwise a reclaimed job sees its own old pointer
// and incorrectly reports an active synchronization.
character = await findCharacter(ctx, job.userId, job.characterId);
if (!character || !authorityMatches(character, job)) throw new Error('Synchronization authorization is stale');
}
if (character.syncGeneration && (character.syncLeaseExpiresAt ?? 0) > now()) throw new Error('Character synchronization is already leased');
const generation = newIdentifier('generation');
await ctx.db.insert('assetGenerations', { userId: job.userId, characterId: job.characterId, generation, status: 'staging', stagedChunkCount: 0, grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion });
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'syncing', grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion, error: undefined } : entry);
await ctx.db.patch(character._id, { state: 'syncing', datasetStatuses: statuses, syncGeneration: generation, syncLeaseExpiresAt: now() + 10 * 60 * 1000, error: undefined });
await ctx.db.patch(job._id, { generation, updatedAt: now() });
return generation;
},
});
export const bridgeStageSyncAssets = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), index: v.number(), assets: v.array(assetValidator) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
if (!Number.isInteger(args.index) || args.index < 0 || args.assets.length > MAX_ASSET_CHUNK) throw new Error('Invalid asset chunk');
const job = await jobLease(ctx, args.jobId, args.leaseToken);
const character = await jobCharacter(ctx, job);
if (args.assets.some((asset: any) => asset.characterId !== job.characterId)) throw new Error('Asset character mismatch');
if (!job.generation || character.syncGeneration !== job.generation) throw new Error('Synchronization generation is not current');
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
if (!generation || generation.status !== 'staging') throw new Error('Snapshot is not staging');
const stagedChunkCount = generation.stagedChunkCount ?? 0;
if (stagedChunkCount >= MAX_CHUNKS || args.index > stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order');
const existing = await ctx.db.query('assetChunks').withIndex('by_snapshot_index', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation).eq('index', args.index)).first();
if (existing) {
// Convex may commit a mutation while its network response is lost. A
// duplicate delivery is accepted only when its payload is identical.
if (JSON.stringify(existing.assets) !== JSON.stringify(args.assets)) throw new Error('Asset chunk already exists');
return null;
}
if (args.index !== stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order');
await ctx.db.insert('assetChunks', { userId: job.userId, characterId: job.characterId, generation: job.generation, index: args.index, assets: args.assets });
await ctx.db.patch(generation._id, { stagedChunkCount: stagedChunkCount + 1 });
return null;
},
});
function validJobSnapshotTimes(syncedAt: number, expiresAt: number, timestamp: number): boolean {
return Number.isFinite(syncedAt) && syncedAt >= 0 && syncedAt <= timestamp && Number.isFinite(expiresAt) && expiresAt > timestamp && expiresAt > syncedAt && expiresAt - syncedAt <= SNAPSHOT_MAX_LIFETIME_MS;
}
export const bridgeCommitSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), chunkCount: v.number(), syncedAt: v.number(), expiresAt: v.number() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const job = await findJob(ctx, args.jobId);
if (job?.status === 'succeeded' && job.generation !== undefined) return true;
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
const character = await jobCharacter(ctx, leased);
if (!leased.generation) throw new Error('Synchronization generation is not current');
if (!Number.isInteger(args.chunkCount) || args.chunkCount < 0 || args.chunkCount > MAX_CHUNKS || !validJobSnapshotTimes(args.syncedAt, args.expiresAt, now())) throw new Error('Invalid snapshot');
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first();
if (!generation || generation.status !== 'staging' || (generation.stagedChunkCount ?? 0) !== args.chunkCount) throw new Error('Snapshot is incomplete');
if (generation.grantRevision !== character.grantRevision || generation.authorizationEpoch !== character.authorizationEpoch || generation.credentialVersion !== character.credentialVersion) throw new Error('Synchronization authorization is stale');
await ctx.db.patch(generation._id, { status: 'complete', chunkCount: args.chunkCount, syncedAt: args.syncedAt, expiresAt: args.expiresAt });
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'ready', generation: leased.generation, syncedAt: args.syncedAt, error: undefined } : entry);
await ctx.db.patch(character._id, { state: 'ready', datasetStatuses: statuses, syncedAt: args.syncedAt, expiresAt: args.expiresAt, activeGeneration: leased.generation, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: undefined });
await ctx.db.patch(leased._id, { status: 'succeeded', updatedAt: now(), finishedAt: now(), leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, cancelRequested: undefined, lastError: undefined });
return true;
},
});
export const bridgeFailSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
const character = await jobCharacter(ctx, leased);
const timestamp = now();
if (leased.generation !== undefined) {
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first();
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: safeJobError() });
}
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: safeJobError() } : entry);
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, error: safeJobError(), syncGeneration: undefined, syncLeaseExpiresAt: undefined });
await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
return true;
},
});
/** Reschedule transient provider/cooldown failures without exposing their raw text. */
export const bridgeDeferSyncJob = mutationGeneric({
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), dueAt: v.number(), cooldownContext: v.optional(v.string()), cooldownUntil: v.optional(v.number()), cooldownGroup: v.optional(v.string()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
const timestamp = now();
if (!Number.isFinite(args.dueAt) || args.dueAt < timestamp || args.dueAt > timestamp + 24 * 60 * 60 * 1000) throw new Error('Invalid retry due time');
if ((leased.attempts ?? 0) >= (leased.maxAttempts ?? 5)) {
await jobCharacter(ctx, leased);
await markTerminalSyncFailure(ctx, leased);
await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
return false;
}
await ctx.db.patch(leased._id, { status: 'queued', updatedAt: timestamp, dueAt: args.dueAt, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization scheduled for retry', ...(args.cooldownGroup ? { cooldownGroup: args.cooldownGroup } : {}) });
if (args.cooldownContext && args.cooldownUntil !== undefined && Number.isFinite(args.cooldownUntil)) {
const contexts = [args.cooldownContext, ...(args.cooldownGroup ? [`group:${args.cooldownGroup}`] : [])];
for (const context of contexts) {
const existing = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first();
const until = Math.max(existing?.until ?? 0, args.cooldownUntil);
if (existing) await ctx.db.patch(existing._id, { until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp });
else await ctx.db.insert('syncCooldowns', { context, until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp });
}
}
return true;
},
});
async function currentCooldown(ctx: any, context: string, group?: string): Promise<any | null> {
const rows = [await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first()];
if (group) rows.push(await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${group}`)).first());
const row = rows.filter(Boolean).sort((left: any, right: any) => right.until - left.until)[0];
return row && row.until > now() ? { until: row.until, group: row.group, kind: row.kind } : null;
}
export const bridgeCooldown = queryGeneric({
args: { ...bridgeArgs, context: v.string() },
handler: async (ctx, args) => {
assertBridge(args.bridgeSecret);
return await currentCooldown(ctx, args.context);
},
});
export const bridgeSyncWorkerCooldown = queryGeneric({
args: { ...workerBridgeArgs, context: v.string(), group: v.optional(v.string()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
return await currentCooldown(ctx, args.context, args.group);
},
});
async function markRefreshUncertainRecord(ctx: any, userId: string, characterId: number, lease: any, character: any, record: any): Promise<boolean> {
if (!lease || !character || !record || record.version !== lease.expectedVersion) {
if (lease) await ctx.db.delete(lease._id);
return false;
}
const epoch = character.authorizationEpoch + 1;
const statuses = DATASET_NAMES.map((dataset) => ({ dataset, status: 'blocked' as const, grantRevision: character.grantRevision, authorizationEpoch: epoch, credentialVersion: character.credentialVersion, invalidatedAt: now(), error: 'Reconnect required' }));
// Retain the last complete generation metadata; authorization status blocks
// reads until reconnect, while a later reconnect can explicitly replace it.
await ctx.db.patch(character._id, { authorizationStatus: 'reconnect_required', authorizationEpoch: epoch, datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Reconnect required', state: 'error' });
await ctx.db.delete(record._id); await ctx.db.delete(lease._id);
await invalidateCharacterJobs(ctx, userId, characterId, now());
return true;
}
export const bridgeMarkRefreshDispatched = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false;
await ctx.db.patch(lease._id, { dispatchStartedAt: now() });
return true;
},
});
/** Durable per-character refresh serialization. The encrypted grant never reaches browsers. */
export const bridgeAcquireRefreshLease = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), ownerId: v.string(), leaseMs: v.optional(v.number()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const character = await findCharacter(ctx, args.userId, args.characterId);
if (!character || character.authorizationStatus !== 'active') throw new Error('Character authorization is not active');
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!record) throw new Error('Character authorization is missing');
const timestamp = now(); const leaseMs = args.leaseMs ?? 2 * 60 * 1000;
if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) throw new Error('Invalid refresh lease duration');
const existing = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (existing && existing.expiresAt > timestamp) return null;
if (existing?.dispatchStartedAt !== undefined) {
await markRefreshUncertainRecord(ctx, args.userId, args.characterId, existing, character, record);
return null;
}
if (existing) await ctx.db.delete(existing._id);
const leaseToken = newIdentifier('refresh');
await ctx.db.insert('refreshLeases', { userId: args.userId, characterId: args.characterId, leaseToken, ownerId: args.ownerId, expiresAt: timestamp + leaseMs, expectedVersion: record.version });
return { leaseToken, expiresAt: timestamp + leaseMs, expectedVersion: record.version, encryptedTokens: record.encryptedTokens, tokenExpiresAt: record.tokenExpiresAt, scopes: record.scopes, grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion };
},
});
export const bridgeRenewRefreshLease = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), leaseMs: v.optional(v.number()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now()) return false;
const leaseMs = args.leaseMs ?? 2 * 60 * 1000;
if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) return false;
await ctx.db.patch(lease._id, { expiresAt: now() + leaseMs }); return true;
},
});
export const bridgeReleaseRefreshLease = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false;
await ctx.db.delete(lease._id);
return true;
},
});
export const bridgeCommitRefresh = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number(), encryptedTokens: v.string(), tokenExpiresAt: v.number(), scopes: v.array(v.string()), jobId: v.optional(v.string()), jobLeaseToken: v.optional(v.string()) },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
const character = await findCharacter(ctx, args.userId, args.characterId);
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt === undefined || !character || character.authorizationStatus !== 'active' || !record || record.version !== args.expectedVersion) return null;
if (!validActualScopes(args.scopes) || !scopesSubset(args.scopes, character.scopes ?? [])) throw new Error('Invalid refreshed scopes');
const nextVersion = record.version + 1;
const nextRevision = character.grantRevision + (scopesEqual(character.scopes, args.scopes) ? 0 : 1);
const updated = updateDatasetStatuses(character, args.scopes, nextRevision, character.authorizationEpoch, nextVersion, now());
await ctx.db.patch(record._id, { encryptedTokens: args.encryptedTokens, tokenExpiresAt: args.tokenExpiresAt, scopes: args.scopes, version: nextVersion });
await ctx.db.patch(character._id, { scopes: args.scopes, grantRevision: nextRevision, credentialVersion: nextVersion, datasetStatuses: updated.statuses, ...(updated.assetsInvalidated ? { state: 'error' as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}) });
if (updated.assetsInvalidated) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now());
else if (args.jobId && args.jobLeaseToken) {
const job = await findJob(ctx, args.jobId);
if (job && job.userId === args.userId && job.characterId === args.characterId && job.status === 'running' && job.leaseToken === args.jobLeaseToken) {
await ctx.db.patch(job._id, { grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: nextVersion, scopes: args.scopes, updatedAt: now() });
}
}
await ctx.db.delete(lease._id);
return { saved: true, credentialVersion: nextVersion, grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, scopes: args.scopes };
},
});
/** Unknown rotating-token outcomes fail closed and require an explicit reconnect. */
export const bridgeMarkRefreshUncertain = mutationGeneric({
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number() },
handler: async (ctx, args) => {
assertSyncWorker(args.workerSecret);
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId) return false;
const character = await findCharacter(ctx, args.userId, args.characterId);
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
if (lease.expectedVersion !== args.expectedVersion) return false;
return await markRefreshUncertainRecord(ctx, args.userId, args.characterId, lease, character, record);
},
});
export const bridgeSaveInteraction = mutationGeneric({ export const bridgeSaveInteraction = mutationGeneric({
args: { args: {
...bridgeArgs, ...bridgeArgs,
+50 -1
View File
@@ -4,6 +4,7 @@ import { v } from "convex/values";
/** The persisted character state is deliberately a small, explicit state machine. */ /** The persisted character state is deliberately a small, explicit state machine. */
export const characterStateValidator = v.union( export const characterStateValidator = v.union(
v.literal("never"), v.literal("never"),
v.literal("queued"),
v.literal("syncing"), v.literal("syncing"),
v.literal("ready"), v.literal("ready"),
v.literal("error"), v.literal("error"),
@@ -16,6 +17,7 @@ export const authorizationStatusValidator = v.union(
v.literal("quarantined"), v.literal("quarantined"),
v.literal("revoked"), v.literal("revoked"),
v.literal("transferred"), v.literal("transferred"),
v.literal("reconnect_required"),
); );
export const datasetStateValidator = v.union( export const datasetStateValidator = v.union(
v.literal("never"), v.literal("never"),
@@ -180,9 +182,56 @@ export default defineSchema({
updatedAt: v.number(), updatedAt: v.number(),
leaseExpiresAt: v.optional(v.number()), leaseExpiresAt: v.optional(v.number()),
characterId: v.optional(v.number()), characterId: v.optional(v.number()),
// Durable sync orchestration fields. They are optional for pre-T07
// placeholder rows; new jobs always populate the complete fence.
dataset: v.optional(v.string()),
dedupeKey: v.optional(v.string()),
dueAt: v.optional(v.number()),
attempts: v.optional(v.number()),
maxAttempts: v.optional(v.number()),
priority: v.optional(v.number()),
lastClaimedAt: v.optional(v.number()),
leaseToken: v.optional(v.string()),
workerId: v.optional(v.string()),
cancelRequested: v.optional(v.boolean()),
cancelledAt: v.optional(v.number()),
startedAt: v.optional(v.number()),
finishedAt: v.optional(v.number()),
lastError: v.optional(v.string()),
generation: v.optional(v.string()),
grantRevision: v.optional(v.number()),
authorizationEpoch: v.optional(v.number()),
credentialVersion: v.optional(v.number()),
scopes: v.optional(v.array(v.string())),
ownerKind: v.optional(v.string()),
authority: v.optional(v.string()),
cooldownGroup: v.optional(v.string()),
}) })
.index("by_user_status", ["userId", "status"]) .index("by_user_status", ["userId", "status"])
.index("by_lease_expires_at", ["leaseExpiresAt"]), .index("by_lease_expires_at", ["leaseExpiresAt"])
.index("by_dedupe", ["dedupeKey"])
.index("by_status_due", ["status", "dueAt"])
.index("by_kind_status_due", ["kind", "status", "dueAt"])
.index("by_asset_sync_queue", ["kind", "dataset", "authority", "ownerKind", "status", "dueAt"])
.index("by_user_asset_status", ["userId", "kind", "dataset", "authority", "ownerKind", "status"]),
refreshLeases: defineTable({
userId: v.string(),
characterId: v.number(),
leaseToken: v.string(),
ownerId: v.string(),
expiresAt: v.number(),
expectedVersion: v.number(),
dispatchStartedAt: v.optional(v.number()),
}).index("by_user_character", ["userId", "characterId"]),
syncCooldowns: defineTable({
context: v.string(),
until: v.number(),
group: v.optional(v.string()),
kind: v.string(),
updatedAt: v.number(),
}).index("by_context", ["context"]).index("by_until", ["until"]),
referenceSyncMetadata: defineTable({ referenceSyncMetadata: defineTable({
source: v.string(), source: v.string(),
+95
View File
@@ -0,0 +1,95 @@
export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined';
export type SyncJob = {
id: string;
dataset: 'assets';
characterId: number;
status: SyncJobStatus;
attempts: number;
dueAt: number;
createdAt: number;
updatedAt: number;
leaseExpiresAt: number | null;
cancelRequested: boolean;
error?: string;
};
export type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' };
export type SyncEnqueueResponse = { queued: number; jobs: SyncJob[]; skipped: SyncSkip[]; partial?: boolean; demo?: boolean };
const JOB_STATUSES = new Set<SyncJobStatus>(['queued', 'running', 'succeeded', 'failed', 'cancelled', 'quarantined']);
const SKIP_REASONS = new Set<SyncSkip['reason']>(['authorization_required', 'queue_full', 'enqueue_failed']);
function parseJob(value: unknown): SyncJob | null {
if (!value || typeof value !== 'object') return null;
const row = value as Record<string, unknown>;
if (typeof row.id !== 'string' || typeof row.characterId !== 'number' || !Number.isSafeInteger(row.characterId) || typeof row.status !== 'string' || !JOB_STATUSES.has(row.status as SyncJobStatus)) return null;
return {
id: row.id, dataset: 'assets', characterId: row.characterId, status: row.status as SyncJobStatus,
attempts: typeof row.attempts === 'number' ? row.attempts : 0,
dueAt: typeof row.dueAt === 'number' ? row.dueAt : 0,
createdAt: typeof row.createdAt === 'number' ? row.createdAt : 0,
updatedAt: typeof row.updatedAt === 'number' ? row.updatedAt : 0,
leaseExpiresAt: typeof row.leaseExpiresAt === 'number' ? row.leaseExpiresAt : null,
cancelRequested: row.cancelRequested === true,
...(typeof row.error === 'string' ? { error: row.error } : {}),
};
}
export function parseSyncEnqueueBody(value: unknown): SyncEnqueueResponse | { error: string } {
if (!value || typeof value !== 'object') return { error: 'Synchronization response was invalid.' };
const body = value as Record<string, unknown>;
if (!Array.isArray(body.jobs) || !Array.isArray(body.skipped)) return { error: typeof body.error === 'string' ? body.error : 'Synchronization response was invalid.' };
const jobs = body.jobs.map(parseJob);
const skipped = body.skipped.map((entry: unknown) => {
if (!entry || typeof entry !== 'object') return null;
const row = entry as Record<string, unknown>;
return typeof row.characterId === 'number' && Number.isSafeInteger(row.characterId) && typeof row.reason === 'string' && SKIP_REASONS.has(row.reason as SyncSkip['reason'])
? { characterId: row.characterId, reason: row.reason as SyncSkip['reason'] } : null;
});
if (jobs.some((job) => job === null) || skipped.some((entry) => entry === null)) return { error: 'Synchronization response was invalid.' };
const queued = typeof body.queued === 'number' && Number.isSafeInteger(body.queued) && body.queued >= 0 ? body.queued : jobs.length;
return { queued, jobs: jobs as SyncJob[], skipped: skipped as SyncSkip[], ...(body.partial === true ? { partial: true } : {}), ...(body.demo === true ? { demo: true } : {}) };
}
export async function enqueueSync(signal?: AbortSignal): Promise<{ response: Response; body: SyncEnqueueResponse | { error: string } }> {
const response = await fetch('/api/sync', signal ? { method: 'POST', signal } : { method: 'POST' });
return { response, body: parseSyncEnqueueBody(await response.json().catch(() => null)) };
}
export const SYNC_STATUS_TIMEOUT_MS = 15_000;
export async function syncJobStatus(id: string, signal?: AbortSignal, timeoutMs = SYNC_STATUS_TIMEOUT_MS): Promise<SyncJob | null> {
const requestController = new AbortController();
let timeout: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
const aborted = new Promise<never>((_, reject) => {
onAbort = () => { requestController.abort(); reject(new DOMException('The operation was aborted', 'AbortError')); };
if (signal?.aborted) onAbort();
else signal?.addEventListener('abort', onAbort, { once: true });
});
const timedOut = new Promise<never>((_, reject) => {
timeout = setTimeout(() => { requestController.abort(); reject(new Error('Synchronization status timed out.')); }, timeoutMs);
});
try {
const response = await Promise.race([fetch(`/api/sync/status?job=${encodeURIComponent(id)}`, { signal: requestController.signal }), timedOut, aborted]);
if (response.status === 404) return null;
if (!response.ok) throw new Error('Synchronization status unavailable.');
return await response.json() as SyncJob;
} finally {
if (timeout !== undefined) clearTimeout(timeout);
if (signal && onAbort) signal.removeEventListener('abort', onAbort);
}
}
export async function cancelSyncJob(id: string): Promise<SyncJob | null> {
const response = await fetch('/api/sync/cancel', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ jobId: id }),
});
if (!response.ok) throw new Error('Could not cancel synchronization.');
return await response.json() as SyncJob;
}
export function syncTerminal(status: SyncJobStatus): boolean {
return status === 'succeeded' || status === 'failed' || status === 'cancelled' || status === 'quarantined';
}
+16 -2
View File
@@ -18,12 +18,12 @@ export type Character = {
scopes: string[]; scopes: string[];
/** Exactly one active primary may authenticate while signed out. */ /** Exactly one active primary may authenticate while signed out. */
loginRole: 'primary' | 'data_only'; loginRole: 'primary' | 'data_only';
authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred'; authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred' | 'reconnect_required';
grantRevision: number; grantRevision: number;
credentialVersion: number; credentialVersion: number;
authorizationEpoch: number; authorizationEpoch: number;
datasets: CharacterDatasetStatus[]; datasets: CharacterDatasetStatus[];
assets: { status: 'never' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string }; assets: { status: 'never' | 'queued' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string };
}; };
export type Asset = { export type Asset = {
itemId: string; itemId: string;
@@ -307,6 +307,20 @@ export type JobOwner =
export type JobAuthority = 'user' | 'system'; export type JobAuthority = 'user' | 'system';
export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined'; export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined';
export type SyncJobDataset = 'assets';
export type SafeSyncJob = {
id: string;
dataset: SyncJobDataset;
characterId: number;
status: SyncJobStatus;
attempts: number;
dueAt: number;
createdAt: number;
updatedAt: number;
leaseExpiresAt: number | null;
cancelRequested: boolean;
error?: string;
};
export type PublicationOutcome = 'success' | 'failure'; export type PublicationOutcome = 'success' | 'failure';
export type PublicationResourceFence = SelectedResource & { export type PublicationResourceFence = SelectedResource & {
/** Monotonic revocation/quarantine epoch, distinct from grant and credential versions. */ /** Monotonic revocation/quarantine epoch, distinct from grant and credential versions. */
+17 -92
View File
@@ -1,10 +1,10 @@
import type { RequestEvent } from '@sveltejs/kit'; import type { RequestEvent } from '@sveltejs/kit';
import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository } from '$lib/domain/model'; import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository, SafeSyncJob } from '$lib/domain/model';
import { AssetService } from '$lib/domain/assets'; import { AssetService } from '$lib/domain/assets';
import { configuration, requireLiveConfig } from './config'; import { configuration, requireLiveConfig } from './config';
import { ConvexStore } from './store'; import { ConvexStore } from './store';
import { TokenCipher, hashSecret } from './security'; import { TokenCipher, hashSecret } from './security';
import { EveSso, type VerifiedGrant } from './sso'; import { EveSso } from './sso';
import { EsiClient, EsiMarketEstimates } from './esi'; import { EsiClient, EsiMarketEstimates } from './esi';
import { SqliteReferenceRepository } from './reference'; import { SqliteReferenceRepository } from './reference';
import { capabilityDisabledReason, capabilityEnabled, type ReleaseCapability } from './release-gate'; import { capabilityDisabledReason, capabilityEnabled, type ReleaseCapability } from './release-gate';
@@ -91,104 +91,29 @@ export async function assetReport(
finally { reference?.close(); } finally { reference?.close(); }
} }
type CommittedGrant = VerifiedGrant & { type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' };
credentialVersion: number;
grantRevision: number;
authorizationEpoch: number;
};
const refreshes = new Map<string, Promise<CommittedGrant>>();
async function characterGrant(userId: string, characterId: number, revalidateOwnership = false): Promise<CommittedGrant> {
const key = `${userId}:${characterId}`;
const existing = refreshes.get(key); if (existing) return existing;
const promise = (async () => {
const { store, cipher, sso } = live();
const record = await store.tokenRecord(userId, characterId);
if (!record) throw new Error('Character authorization is missing');
const previous = await cipher.open<VerifiedGrant>(record.encryptedTokens, `character:${characterId}`);
if (previous.characterId !== characterId) throw new Error('Character authorization mismatch');
if (!revalidateOwnership && previous.expiresAt > Date.now() + 60_000) {
const current = (await store.characters(userId)).find((entry) => entry.id === characterId);
if (!current) throw new Error('Character authorization is missing');
return {
...previous,
credentialVersion: current.credentialVersion,
grantRevision: current.grantRevision,
authorizationEpoch: current.authorizationEpoch,
};
}
const grant = await sso.refresh(previous.refreshToken);
if (grant.characterId !== characterId || grant.ownerHash !== previous.ownerHash) {
await store.quarantineAuthorization(userId, characterId).catch(() => {});
throw new Error('Character ownership changed; reconnect required');
}
const saved = await store.saveTokens(userId, characterId, { encryptedTokens: await cipher.seal(grant, `character:${characterId}`), tokenExpiresAt: grant.expiresAt, scopes: grant.scopes, expectedVersion: record.version });
if (!saved) throw new Error('Authorization changed concurrently; retry synchronization');
return {
...grant,
credentialVersion: saved.credentialVersion,
grantRevision: saved.grantRevision,
authorizationEpoch: saved.authorizationEpoch,
scopes: saved.scopes,
};
})(); export async function synchronize(userId: string, initiatingSessionHash?: string) {
refreshes.set(key, promise);
try { return await promise; } finally { if (refreshes.get(key) === promise) refreshes.delete(key); }
}
export async function synchronize(userId: string) {
const config = configuration(); const config = configuration();
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) { if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) {
throw new Error(capabilityDisabledReason('asset_sync')); throw new Error(capabilityDisabledReason('asset_sync'));
} }
if (config.mode === 'demo') return { completed: 2, failed: [] as number[], demo: true }; if (config.mode === 'demo') return { queued: 2, jobs: [] as SafeSyncJob[], skipped: [] as SyncSkip[], partial: false, demo: true };
const { store, esi } = live(); if (!initiatingSessionHash) throw new Error('Authenticated session proof is required');
const { store } = live();
const characters = await store.characters(userId); const characters = await store.characters(userId);
const failed: number[] = []; let completed = 0; const jobs: SafeSyncJob[] = [];
const skipped: SyncSkip[] = [];
for (const character of characters) { for (const character of characters) {
if (character.authorizationStatus !== 'active') { failed.push(character.id); continue; } if (character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) {
if (!character.scopes.includes('esi-assets.read_assets.v1')) { failed.push(character.id); continue; } skipped.push({ characterId: character.id, reason: 'authorization_required' });
let generation: string | undefined; continue;
}
try { try {
// Refresh first so the generation fence records the credential version jobs.push(await store.enqueueAssetSync(userId, character.id, initiatingSessionHash));
// that actually authorizes the ESI request and publication. } catch (error) {
const grant = await characterGrant(userId, character.id, true); skipped.push({ characterId: character.id, reason: error instanceof Error && error.message === 'Synchronization queue is full' ? 'queue_full' : 'enqueue_failed' });
if (!grant.scopes.includes('esi-assets.read_assets.v1')) throw new Error('Asset authorization missing');
// Refresh/CAS may have advanced authority while the initial character
// list was in memory. Reload before leasing and bind beginSync to the
// exact revisions/scopes that authorize the transport request.
const current = (await store.characters(userId)).find((entry) => entry.id === character.id);
if (!current || current.authorizationStatus !== 'active'
|| !current.scopes.includes('esi-assets.read_assets.v1')
|| current.credentialVersion !== grant.credentialVersion
|| current.grantRevision !== grant.grantRevision
|| current.authorizationEpoch !== grant.authorizationEpoch
|| JSON.stringify([...current.scopes].sort()) !== JSON.stringify([...grant.scopes].sort())) {
throw new Error('Asset authorization changed after credential commit');
}
const authority = {
userId,
principal: grant.ownerHash,
characterId: current.id,
scopes: current.scopes,
grantRevision: grant.grantRevision,
authorizationEpoch: grant.authorizationEpoch,
credentialVersion: grant.credentialVersion,
} as const;
generation = await store.beginSync(userId, current.id, {
scopes: authority.scopes,
grantRevision: authority.grantRevision,
authorizationEpoch: authority.authorizationEpoch,
credentialVersion: authority.credentialVersion,
});
const snapshot = await esi.getAssets(current.id, grant.accessToken, authority);
const chunks = Math.ceil(snapshot.assets.length / 250);
for (let index = 0; index < chunks; index++) await store.stageAssets(userId, character.id, generation, index, snapshot.assets.slice(index * 250, (index + 1) * 250));
await store.finishSync(userId, character.id, generation, chunks, snapshot.syncedAt, snapshot.expiresAt);
completed++;
} catch {
failed.push(character.id);
if (generation) await store.failSync(userId, character.id, generation, 'Synchronization failed. Reconnect if authorization was revoked, or retry after ESI recovers.').catch(() => {});
} }
} }
return { completed, failed, demo: false }; return { queued: jobs.length, jobs, skipped, partial: skipped.length > 0, demo: false };
} }
+153 -7
View File
@@ -5,6 +5,7 @@ import type {
Character, Character,
CharacterAssetRepository, CharacterAssetRepository,
UserId, UserId,
SafeSyncJob,
} from "$lib/domain/model"; } from "$lib/domain/model";
export type OAuthStateRecord = { export type OAuthStateRecord = {
@@ -67,6 +68,31 @@ export type SavedInteraction = SaveInteractionInput & {
created: boolean; created: boolean;
}; };
export type ClaimedSyncJob = SafeSyncJob & {
jobId: string;
leaseToken: string;
userId: string;
characterId: number;
grantRevision: number;
authorizationEpoch: number;
credentialVersion: number;
scopes: string[];
generation?: string;
cooldownGroup?: string;
};
export type RefreshLease = {
leaseToken: string;
expiresAt: number;
expectedVersion: number;
encryptedTokens: string;
tokenExpiresAt: number;
scopes: string[];
grantRevision: number;
authorizationEpoch: number;
credentialVersion: number;
};
type AssetPage = { type AssetPage = {
page: Asset[]; page: Asset[];
isDone: boolean; isDone: boolean;
@@ -83,6 +109,9 @@ const QUERY = {
tokenRecord: "functions:bridgeTokenRecord", tokenRecord: "functions:bridgeTokenRecord",
characters: "functions:bridgeCharacters", characters: "functions:bridgeCharacters",
assetsPage: "functions:bridgeAssetsPage", assetsPage: "functions:bridgeAssetsPage",
syncJob: "functions:bridgeSyncJob",
cooldown: "functions:bridgeCooldown",
workerCooldown: "functions:bridgeSyncWorkerCooldown",
} as const; } as const;
const MUTATION = { const MUTATION = {
@@ -99,6 +128,23 @@ const MUTATION = {
setPrimary: "functions:bridgeSetPrimary", setPrimary: "functions:bridgeSetPrimary",
invalidateAuthorization: "functions:bridgeInvalidateAuthorization", invalidateAuthorization: "functions:bridgeInvalidateAuthorization",
quarantineAuthorization: "functions:bridgeQuarantineAuthorization", quarantineAuthorization: "functions:bridgeQuarantineAuthorization",
enqueueAssetSync: "functions:bridgeEnqueueAssetSync",
cancelSyncJob: "functions:bridgeCancelSyncJob",
claimSyncJob: "functions:bridgeClaimSyncJob",
renewSyncJob: "functions:bridgeRenewSyncJob",
beginSyncJob: "functions:bridgeBeginSyncJob",
refreshSyncJobAuthority: "functions:bridgeRefreshSyncJobAuthority",
restartSyncJobGeneration: "functions:bridgeRestartSyncJobGeneration",
stageSyncAssets: "functions:bridgeStageSyncAssets",
commitSyncJob: "functions:bridgeCommitSyncJob",
failSyncJob: "functions:bridgeFailSyncJob",
deferSyncJob: "functions:bridgeDeferSyncJob",
acquireRefreshLease: "functions:bridgeAcquireRefreshLease",
renewRefreshLease: "functions:bridgeRenewRefreshLease",
releaseRefreshLease: "functions:bridgeReleaseRefreshLease",
commitRefresh: "functions:bridgeCommitRefresh",
markRefreshUncertain: "functions:bridgeMarkRefreshUncertain",
markRefreshDispatched: "functions:bridgeMarkRefreshDispatched",
} as const; } as const;
function queryReference(name: string): any { function queryReference(name: string): any {
@@ -110,28 +156,37 @@ function mutationReference(name: string): any {
} }
/** /**
* Server-side Convex repository. Every call goes through a public bridge * Server-side Convex repository. Web calls use the general bridge secret;
* function carrying the dedicated bridge secret; this adapter never uses a * worker lifecycle calls use a separate narrowly-scoped worker secret. This
* deployment admin key and is not safe to import into browser code. * adapter never uses a deployment admin key and is not safe to import into
* browser code.
*/ */
export class ConvexStore implements CharacterAssetRepository { export class ConvexStore implements CharacterAssetRepository {
private readonly client: ConvexHttpClient; private readonly client: ConvexHttpClient;
private readonly bridgeSecret: string; private readonly bridgeSecret?: string;
private readonly workerSecret?: string;
public constructor(url: string, bridgeSecret: string) { public constructor(url: string, bridgeSecret?: string, workerSecret?: string) {
if (!url) throw new Error("Convex URL is required"); if (!url) throw new Error("Convex URL is required");
if (!bridgeSecret) throw new Error("Convex bridge secret is required"); if (!bridgeSecret && !workerSecret) throw new Error("Convex bridge or worker secret is required");
this.bridgeSecret = bridgeSecret;
this.workerSecret = workerSecret;
this.client = new ConvexHttpClient(url, { this.client = new ConvexHttpClient(url, {
skipConvexDeploymentUrlCheck: true, skipConvexDeploymentUrlCheck: true,
logger: false, logger: false,
}); });
this.bridgeSecret = bridgeSecret;
} }
private bridgeArgs(args: Record<string, unknown>): Record<string, unknown> { private bridgeArgs(args: Record<string, unknown>): Record<string, unknown> {
if (!this.bridgeSecret) throw new Error("General bridge authority is unavailable to this store");
return { ...args, bridgeSecret: this.bridgeSecret }; return { ...args, bridgeSecret: this.bridgeSecret };
} }
private workerArgs(args: Record<string, unknown>): Record<string, unknown> {
if (!this.workerSecret) throw new Error("Worker bridge authority is unavailable to this store");
return { ...args, workerSecret: this.workerSecret };
}
private async query<T>(name: string, args: Record<string, unknown>): Promise<T> { private async query<T>(name: string, args: Record<string, unknown>): Promise<T> {
return (await this.client.query( return (await this.client.query(
queryReference(name), queryReference(name),
@@ -147,6 +202,21 @@ export class ConvexStore implements CharacterAssetRepository {
)) as T; )) as T;
} }
private async workerMutation<T>(name: string, args: Record<string, unknown>): Promise<T> {
return (await this.client.mutation(
mutationReference(name),
this.workerArgs(args),
{ skipQueue: true },
)) as T;
}
private async workerQuery<T>(name: string, args: Record<string, unknown>): Promise<T> {
return (await this.client.query(
queryReference(name),
this.workerArgs(args),
)) as T;
}
public async session(hash: string): Promise<{ userId: string; sessionId: string } | null> { public async session(hash: string): Promise<{ userId: string; sessionId: string } | null> {
return await this.query<{ userId: string; sessionId: string } | null>(QUERY.session, { return await this.query<{ userId: string; sessionId: string } | null>(QUERY.session, {
sessionHash: hash, sessionHash: hash,
@@ -294,6 +364,82 @@ export class ConvexStore implements CharacterAssetRepository {
throw new Error(SNAPSHOT_CHANGED); throw new Error(SNAPSHOT_CHANGED);
} }
public async enqueueAssetSync(userId: string, characterId: number, initiatingSessionHash: string): Promise<SafeSyncJob> {
return await this.mutation<SafeSyncJob>(MUTATION.enqueueAssetSync, { userId, characterId, initiatingSessionHash });
}
public async syncJob(userId: string, jobId: string): Promise<SafeSyncJob | null> {
return await this.query<SafeSyncJob | null>(QUERY.syncJob, { userId, jobId });
}
public async cancelSyncJob(userId: string, jobId: string, initiatingSessionHash: string): Promise<SafeSyncJob | null> {
return await this.mutation<SafeSyncJob | null>(MUTATION.cancelSyncJob, { userId, jobId, initiatingSessionHash });
}
public async claimSyncJob(workerId: string, leaseMs?: number): Promise<ClaimedSyncJob | null> {
return await this.workerMutation<ClaimedSyncJob | null>(MUTATION.claimSyncJob, { workerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
}
public async renewSyncJob(jobId: string, leaseToken: string, workerId: string, leaseMs?: number): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.renewSyncJob, { jobId, leaseToken, workerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
}
public async beginSyncJob(jobId: string, leaseToken: string): Promise<string> {
return await this.workerMutation<string>(MUTATION.beginSyncJob, { jobId, leaseToken });
}
public async refreshSyncJobAuthority(jobId: string, leaseToken: string, expected: { grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }): Promise<{ grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }> {
return await this.workerMutation(MUTATION.refreshSyncJobAuthority, { jobId, leaseToken, expectedGrantRevision: expected.grantRevision, expectedAuthorizationEpoch: expected.authorizationEpoch, expectedCredentialVersion: expected.credentialVersion, expectedScopes: expected.scopes });
}
public async restartSyncJobGeneration(jobId: string, leaseToken: string): Promise<void> {
await this.workerMutation(MUTATION.restartSyncJobGeneration, { jobId, leaseToken });
}
public async stageSyncAssets(jobId: string, leaseToken: string, index: number, assets: Asset[]): Promise<void> {
await this.workerMutation(MUTATION.stageSyncAssets, { jobId, leaseToken, index, assets });
}
public async commitSyncJob(jobId: string, leaseToken: string, chunkCount: number, syncedAt: number, expiresAt: number): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.commitSyncJob, { jobId, leaseToken, chunkCount, syncedAt, expiresAt });
}
public async failSyncJob(jobId: string, leaseToken: string): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.failSyncJob, { jobId, leaseToken });
}
public async deferSyncJob(jobId: string, leaseToken: string, dueAt: number, cooldown?: { context: string; until: number; group?: string }): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.deferSyncJob, { jobId, leaseToken, dueAt, ...(cooldown === undefined ? {} : { cooldownContext: cooldown.context, cooldownUntil: cooldown.until, ...(cooldown.group === undefined ? {} : { cooldownGroup: cooldown.group }) }) });
}
public async cooldown(context: string, group?: string): Promise<{ until: number; group?: string; kind: string } | null> {
return await this.workerQuery<{ until: number; group?: string; kind: string } | null>(QUERY.workerCooldown, { context, ...(group === undefined ? {} : { group }) });
}
public async acquireRefreshLease(userId: string, characterId: number, ownerId: string, leaseMs?: number): Promise<RefreshLease | null> {
return await this.workerMutation<RefreshLease | null>(MUTATION.acquireRefreshLease, { userId, characterId, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
}
public async renewRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string, leaseMs?: number): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.renewRefreshLease, { userId, characterId, leaseToken, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
}
public async releaseRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.releaseRefreshLease, { userId, characterId, leaseToken, ownerId });
}
public async markRefreshDispatched(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.markRefreshDispatched, { userId, characterId, leaseToken, ownerId });
}
public async commitRefresh(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number, input: { encryptedTokens: string; tokenExpiresAt: number; scopes: string[]; jobId?: string; jobLeaseToken?: string }): Promise<SavedTokenAuthority | null> {
return await this.workerMutation<SavedTokenAuthority | null>(MUTATION.commitRefresh, { userId, characterId, leaseToken, ownerId, expectedVersion, ...input });
}
public async markRefreshUncertain(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number): Promise<boolean> {
return await this.workerMutation<boolean>(MUTATION.markRefreshUncertain, { userId, characterId, leaseToken, ownerId, expectedVersion });
}
public async beginSync( public async beginSync(
userId: string, userId: string,
characterId: number, characterId: number,
+50 -11
View File
@@ -1,6 +1,7 @@
<script lang="ts"> <script lang="ts">
import { onMount } from 'svelte'; import { onDestroy, onMount } from 'svelte';
import AssetResult from '$lib/components/AssetResult.svelte'; import AssetResult from '$lib/components/AssetResult.svelte';
import { cancelSyncJob, enqueueSync, syncJobStatus, syncTerminal, type SyncJob, type SyncSkip } from '$lib/client/sync-queue';
import type { AssetOperation } from '$lib/domain/model'; import type { AssetOperation } from '$lib/domain/model';
let { data } = $props(); let { data } = $props();
let prompt = $state(''); let prompt = $state('');
@@ -10,29 +11,66 @@
let message = $state(''); let message = $state('');
let failure = $state(''); let failure = $state('');
let operation = $state<AssetOperation | null>(null); let operation = $state<AssetOperation | null>(null);
let syncJobs = $state<SyncJob[]>([]);
let syncSkipped = $state<SyncSkip[]>([]);
let pollController: AbortController | null = null;
let enqueueController: AbortController | null = null;
let requestController: AbortController | null = null;
let pollTimer: ReturnType<typeof setTimeout> | undefined;
function stopPolling() { pollController?.abort(); pollController = null; if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; }
async function pollSync(ids: string[]) {
stopPolling();
const controller = new AbortController(); pollController = controller;
try {
for (let attempt = 0; attempt < 120 && !controller.signal.aborted; attempt += 1) {
const next = await Promise.all(ids.map((id) => syncJobStatus(id, controller.signal)));
if (controller.signal.aborted) return;
if (next.some((job) => job === null)) { busy = false; failure = 'A synchronization job disappeared before completion; no report retry was performed.'; return; }
syncJobs = next as SyncJob[];
if (syncJobs.every((job) => syncTerminal(job.status))) {
busy = false;
if (syncJobs.length > 0 && syncJobs.every((job) => job.status === 'succeeded') && syncSkipped.every((entry) => entry.reason === 'authorization_required')) await ask(submitted, true);
return;
}
await new Promise<void>((resolve) => {
const done = () => { if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; controller.signal.removeEventListener('abort', done); resolve(); };
pollTimer = setTimeout(done, 1_000); controller.signal.addEventListener('abort', done, { once: true });
});
}
if (!controller.signal.aborted) { busy = false; failure = 'Synchronization is still running; retry the report after the queued jobs finish.'; }
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization status unavailable.'; } }
finally { if (pollController === controller) pollController = null; }
}
onDestroy(() => { stopPolling(); enqueueController?.abort(); enqueueController = null; requestController?.abort(); requestController = null; });
async function cancelSync(id: string) {
try { const job = await cancelSyncJob(id); if (job) syncJobs = syncJobs.map((entry) => entry.id === job.id ? job : entry); }
catch (error) { failure = error instanceof Error ? error.message : 'Could not cancel synchronization.'; }
}
async function ask(text = prompt, resume = false) { async function ask(text = prompt, resume = false) {
if (!text.trim() || busy) return; if (!text.trim() || busy) return;
busy = true; failure = ''; submitted = text; busy = true; failure = ''; submitted = text;
requestController?.abort(); const controller = new AbortController(); requestController = controller;
if (!resume || !requestId) requestId = crypto.randomUUID(); if (!resume || !requestId) requestId = crypto.randomUUID();
sessionStorage.setItem('eve-pending', JSON.stringify({ text, requestId })); sessionStorage.setItem('eve-pending', JSON.stringify({ text, requestId }));
try { try {
const response = await fetch('/api/chat', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: text, requestId }) }); const response = await fetch('/api/chat', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: text, requestId }), signal: controller.signal });
const body = await response.json(); const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Request failed'); if (!response.ok) throw new Error(body.error ?? 'Request failed');
message = body.message; operation = body.operation ?? null; message = body.message; operation = body.operation ?? null;
if (!operation || operation.status === 'ready') sessionStorage.removeItem('eve-pending'); if (!operation || operation.status === 'ready') sessionStorage.removeItem('eve-pending');
} catch (error) { failure = error instanceof Error ? error.message : 'Request failed'; } } catch (error) { if (!controller.signal.aborted) failure = error instanceof Error ? error.message : 'Request failed'; }
finally { busy = false; } finally { if (requestController === controller) requestController = null; busy = false; }
} }
async function sync() { async function sync() {
busy = true; failure = ''; stopPolling(); enqueueController?.abort(); const controller = new AbortController(); enqueueController = controller; busy = true; failure = '';
try { try {
const response = await fetch('/api/sync', { method: 'POST' }); const { response, body } = await enqueueSync(controller.signal);
const body = await response.json(); if ('jobs' in body) { syncJobs = body.jobs; syncSkipped = body.skipped; }
if (!response.ok) throw new Error(body.error ?? 'Synchronization failed'); if (!response.ok || !('jobs' in body)) throw new Error(('error' in body && body.error) || (syncSkipped.some((entry) => entry.reason === 'queue_full') ? 'Synchronization queue is full.' : 'Synchronization could not be queued.'));
busy = false; await ask(submitted, true); if (syncJobs.length > 0) await pollSync(syncJobs.map((job) => job.id));
} catch (error) { failure = error instanceof Error ? error.message : 'Synchronization failed'; } else busy = false;
finally { busy = false; } } catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization failed'; } }
finally { if (enqueueController === controller) enqueueController = null; }
} }
onMount(() => { onMount(() => {
try { try {
@@ -54,6 +92,7 @@
</form> </form>
<div class="row"><button class="quiet" disabled={busy} onclick={() => ask('/value')}>Estimate all assets</button><button class="quiet" disabled={busy} onclick={() => ask('/assets')}>Browse assets</button></div> <div class="row"><button class="quiet" disabled={busy} onclick={() => ask('/value')}>Estimate all assets</button><button class="quiet" disabled={busy} onclick={() => ask('/assets')}>Browse assets</button></div>
{#if failure}<p class="error" role="alert">{failure}</p>{/if} {#if failure}<p class="error" role="alert">{failure}</p>{/if}
{#if syncJobs.length || syncSkipped.length}<section class="panel" aria-live="polite"><h2>Synchronization status</h2>{#each syncJobs as job}<p class="row spread"><span>Character {job.characterId}: {job.status}{job.error ? ` — ${job.error}` : ''}</span>{#if job.status === 'queued' || job.status === 'running'}<button class="quiet" onclick={() => cancelSync(job.id)}>Cancel</button>{/if}</p>{/each}{#each syncSkipped as entry}<p class="muted">Character {entry.characterId}: {entry.reason === 'authorization_required' ? 'authorization required' : entry.reason === 'queue_full' ? 'queue full' : 'enqueue failed'}</p>{/each}</section>{/if}
<div aria-live="polite" aria-busy={busy}> <div aria-live="polite" aria-busy={busy}>
{#if submitted}<p class="message">{submitted}</p>{/if} {#if submitted}<p class="message">{submitted}</p>{/if}
{#if message}<p>{message}</p>{/if} {#if message}<p>{message}</p>{/if}
+8 -3
View File
@@ -4,6 +4,8 @@ import { session, synchronize } from '$lib/server/application';
import { protectMutation } from '$lib/server/http'; import { protectMutation } from '$lib/server/http';
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate'; import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
import { configuration } from '$lib/server/config'; import { configuration } from '$lib/server/config';
import { hashSecret } from '$lib/server/security';
import { SESSION_COOKIE } from '$lib/server/application';
export const POST: RequestHandler = async (event) => { export const POST: RequestHandler = async (event) => {
const config = configuration(); const config = configuration();
// Bound origin/session work by the existing anonymous IP bucket first. The // Bound origin/session work by the existing anonymous IP bucket first. The
@@ -16,8 +18,11 @@ export const POST: RequestHandler = async (event) => {
if (current) protectMutation(event, `sync:${current.userId}`, 6); if (current) protectMutation(event, `sync:${current.userId}`, 6);
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 }); if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
try { try {
const result = await synchronize(current.userId); const cookie = event.cookies.get(SESSION_COOKIE);
if (result.failed.length) return json({ ...result, error: `${result.failed.length} character(s) could not synchronize. Previous complete snapshots were preserved; check Characters.` }, { status: 503 }); if (!cookie) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
return json(result); const result = await synchronize(current.userId, hashSecret(cookie));
const failures = result.skipped.filter((entry) => entry.reason !== 'authorization_required');
const status = result.queued > 0 ? 202 : failures.some((entry) => entry.reason === 'queue_full') ? 429 : failures.length > 0 ? 503 : 200;
return json(result, { status });
} catch { return json({ error: 'Synchronization unavailable. Check backend configuration and try again.' }, { status: 503 }); } } catch { return json({ error: 'Synchronization unavailable. Check backend configuration and try again.' }, { status: 503 }); }
}; };
+27
View File
@@ -0,0 +1,27 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { live, session, SESSION_COOKIE } from '$lib/server/application';
import { hashSecret } from '$lib/server/security';
import { configuration } from '$lib/server/config';
import { boundedJson, protectMutation } from '$lib/server/http';
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
export const POST: RequestHandler = async (event) => {
const config = configuration();
protectMutation(event, `sync-cancel:${event.getClientAddress()}`, 6);
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) return json({ error: capabilityDisabledReason('asset_sync') }, { status: 503 });
const current = await session(event);
if (current) protectMutation(event, `sync-cancel:${current.userId}`, 6);
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
const body = await boundedJson(event) as { jobId?: unknown } | null;
const jobId = body && typeof body.jobId === 'string' ? body.jobId : '';
if (!jobId || jobId.length > 128) return json({ error: 'A sync job is required.' }, { status: 400 });
const cookie = event.cookies.get(SESSION_COOKIE);
if (!cookie) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
try {
const result = await live().store.cancelSyncJob(current.userId, jobId, hashSecret(cookie));
return result ? json(result) : json({ error: 'Sync job not found.' }, { status: 404 });
} catch {
return json({ error: 'Synchronization cancellation unavailable.' }, { status: 503 });
}
};
+20
View File
@@ -0,0 +1,20 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { live, session } from '$lib/server/application';
import { configuration } from '$lib/server/config';
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
export const GET: RequestHandler = async (event) => {
const config = configuration();
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) return json({ error: capabilityDisabledReason('asset_sync') }, { status: 503 });
const current = await session(event);
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
const jobId = event.url.searchParams.get('job');
if (!jobId || jobId.length > 128) return json({ error: 'A sync job is required.' }, { status: 400 });
try {
const job = await live().store.syncJob(current.userId, jobId);
return job ? json(job) : json({ error: 'Sync job not found.' }, { status: 404 });
} catch {
return json({ error: 'Synchronization status unavailable.' }, { status: 503 });
}
};
+48 -8
View File
@@ -1,20 +1,60 @@
<script lang="ts"> <script lang="ts">
import { onDestroy } from 'svelte';
import { invalidateAll } from '$app/navigation'; import { invalidateAll } from '$app/navigation';
import AssetResult from '$lib/components/AssetResult.svelte'; import AssetResult from '$lib/components/AssetResult.svelte';
import { cancelSyncJob, enqueueSync, syncJobStatus, syncTerminal, type SyncJob, type SyncSkip } from '$lib/client/sync-queue';
let { data } = $props(); let { data } = $props();
let busy = $state(false); let failure = $state(''); let busy = $state(false); let failure = $state('');
async function sync() { let jobs = $state<SyncJob[]>([]); let skipped = $state<SyncSkip[]>([]);
busy = true; failure = ''; let pollController: AbortController | null = null;
let enqueueController: AbortController | null = null;
let pollTimer: ReturnType<typeof setTimeout> | undefined;
function stopPolling() { pollController?.abort(); pollController = null; if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; }
async function poll(ids: string[]) {
stopPolling();
const controller = new AbortController(); pollController = controller;
try { try {
const response = await fetch('/api/sync', { method: 'POST' }); for (let attempt = 0; attempt < 120 && !controller.signal.aborted; attempt += 1) {
const body = await response.json(); const next = await Promise.all(ids.map((id) => syncJobStatus(id, controller.signal)));
if (!response.ok) throw new Error(body.error ?? 'Synchronization failed'); if (controller.signal.aborted) return;
await invalidateAll(); if (next.some((job) => job === null)) { busy = false; failure = 'A synchronization job disappeared before completion; no report retry was performed.'; return; }
} catch (error) { failure = error instanceof Error ? error.message : 'Synchronization failed'; } jobs = next as SyncJob[];
finally { busy = false; } if (jobs.every((job) => syncTerminal(job.status))) {
busy = false;
if (jobs.length > 0 && jobs.every((job) => job.status === 'succeeded') && skipped.every((entry) => entry.reason === 'authorization_required')) await invalidateAll();
return;
}
await new Promise<void>((resolve) => {
const done = () => { if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; controller.signal.removeEventListener('abort', done); resolve(); };
pollTimer = setTimeout(done, 1_000); controller.signal.addEventListener('abort', done, { once: true });
});
}
if (!controller.signal.aborted) { busy = false; failure = 'Synchronization is still running; this page will retain its queued status.'; }
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization status unavailable.'; } }
finally { if (pollController === controller) pollController = null; }
}
onDestroy(() => { stopPolling(); enqueueController?.abort(); enqueueController = null; });
async function sync() {
stopPolling(); enqueueController?.abort(); const controller = new AbortController(); enqueueController = controller; busy = true; failure = '';
try {
const { response, body } = await enqueueSync(controller.signal);
if ('jobs' in body) { jobs = body.jobs; skipped = body.skipped; }
if (!response.ok || !('jobs' in body)) throw new Error(('error' in body && body.error) || (skipped.some((entry) => entry.reason === 'queue_full') ? 'Synchronization queue is full.' : 'Synchronization could not be queued.'));
if (jobs.length > 0) await poll(jobs.map((job) => job.id));
else busy = false;
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization failed'; } }
finally { if (enqueueController === controller) enqueueController = null; }
}
async function cancel(id: string) {
try { const job = await cancelSyncJob(id); if (job) jobs = jobs.map((entry) => entry.id === job.id ? job : entry); }
catch (error) { failure = error instanceof Error ? error.message : 'Could not cancel synchronization.'; }
}
function skipLabel(reason: SyncSkip['reason']): string {
return reason === 'authorization_required' ? 'authorization required' : reason === 'queue_full' ? 'queue full' : 'enqueue failed';
} }
</script> </script>
<div class="row spread"><div><p class="eyebrow">SHARED DOMAIN VIEW</p><h1>Assets</h1></div><button class="quiet" disabled={busy} onclick={sync}>{busy ? 'Synchronizing…' : 'Synchronize characters'}</button></div> <div class="row spread"><div><p class="eyebrow">SHARED DOMAIN VIEW</p><h1>Assets</h1></div><button class="quiet" disabled={busy} onclick={sync}>{busy ? 'Synchronizing…' : 'Synchronize characters'}</button></div>
<p class="muted">All connected characters. The same deterministic report used by the assistant.</p> <p class="muted">All connected characters. The same deterministic report used by the assistant.</p>
{#if failure}<p class="error" role="alert">{failure}</p>{/if} {#if failure}<p class="error" role="alert">{failure}</p>{/if}
{#if jobs.length || skipped.length}<section class="panel" aria-live="polite"><h2>Synchronization status</h2>{#each jobs as job}<p class="row spread"><span>Character {job.characterId}: {job.status}{job.error ? ` — ${job.error}` : ''}</span>{#if job.status === 'queued' || job.status === 'running'}<button class="quiet" onclick={() => cancel(job.id)}>Cancel</button>{/if}</p>{/each}{#each skipped as entry}<p class="muted">Character {entry.characterId}: {skipLabel(entry.reason)}</p>{/each}</section>{/if}
<AssetResult result={data.result} onSync={sync} /> <AssetResult result={data.result} onSync={sync} />
+19 -1
View File
@@ -20,6 +20,10 @@ describe('Infisical boundary', () => {
test('accepts Bun-forwarded commands and preserves child --help arguments', () => { test('accepts Bun-forwarded commands and preserves child --help arguments', () => {
expect(parseArguments(['run', 'docker', 'compose', 'config', '--quiet']).command).toEqual(['docker', 'compose', 'config', '--quiet']); expect(parseArguments(['run', 'docker', 'compose', 'config', '--quiet']).command).toEqual(['docker', 'compose', 'config', '--quiet']);
expect(parseArguments(['run', '--deployment', '--', 'bun', 'tool.ts', '--help'])).toMatchObject({ deployment: true, command: ['bun', 'tool.ts', '--help'] }); expect(parseArguments(['run', '--deployment', '--', 'bun', 'tool.ts', '--help'])).toMatchObject({ deployment: true, command: ['bun', 'tool.ts', '--help'] });
expect(parseArguments(['run', '--worker', '--', 'bun', 'workers/sync.ts'])).toMatchObject({ worker: true, command: ['bun', 'workers/sync.ts'] });
expect(() => parseArguments(['run', '--configure', '--', 'bun', 'scripts/configure-convex.ts'])).toThrow('--configure requires --deployment');
expect(parseArguments(['run', '--deployment', '--configure', '--', 'bun', 'scripts/configure-convex.ts'])).toMatchObject({ deployment: true, configure: true, command: ['bun', 'scripts/configure-convex.ts'] });
expect(() => parseArguments(['run', '--worker', '--deployment', '--', 'bun', 'worker.ts'])).toThrow();
expect(parseArguments(['status']).action).toBe('status'); expect(parseArguments(['status']).action).toBe('status');
expect(() => parseArguments(['run', '--unknown'])).toThrow(); expect(() => parseArguments(['run', '--unknown'])).toThrow();
expect(() => parseArguments(['status', '--', 'printenv'])).toThrow(); expect(() => parseArguments(['status', '--', 'printenv'])).toThrow();
@@ -87,10 +91,11 @@ describe('Infisical boundary', () => {
test('child processes receive only managed app keys, without bootstrap or ambient admin credentials', () => { test('child processes receive only managed app keys, without bootstrap or ambient admin credentials', () => {
const base = { PATH: '/usr/bin', INFISICAL_TOKEN: 'token', [ADMIN_KEY]: 'ambient-admin', ...credentials, TYPESAFE_API_KEY: 'stale-key' }; const base = { PATH: '/usr/bin', INFISICAL_TOKEN: 'token', [ADMIN_KEY]: 'ambient-admin', ...credentials, TYPESAFE_API_KEY: 'stale-key' };
const values = { EVE_CLIENT_ID: 'client', EVE_CLIENT_SECRET: 'secret', [ADMIN_KEY]: 'vault-admin', NODE_OPTIONS: 'not-injected' }; const values = { EVE_CLIENT_ID: 'client', EVE_CLIENT_SECRET: 'secret', CONVEX_BRIDGE_SECRET: 'bridge', CONVEX_SYNC_WORKER_SECRET: 'worker', APP_AUTH_PRIVATE_KEY: 'private', APP_AUTH_JWKS: 'jwks', [ADMIN_KEY]: 'vault-admin', NODE_OPTIONS: 'not-injected' };
const config = { APP_ORIGIN: 'https://dev.evebot.app' }; const config = { APP_ORIGIN: 'https://dev.evebot.app' };
const env = applicationEnvironment(base, values, config); const env = applicationEnvironment(base, values, config);
expect(env.EVE_CLIENT_SECRET).toBe('secret'); expect(env.EVE_CLIENT_SECRET).toBe('secret');
expect(env.CONVEX_SYNC_WORKER_SECRET).toBeUndefined();
expect(env.PATH).toBe('/usr/bin'); expect(env.PATH).toBe('/usr/bin');
expect(env.APP_ORIGIN).toBe(config.APP_ORIGIN); expect(env.APP_ORIGIN).toBe(config.APP_ORIGIN);
expect(env.TYPESAFE_API_KEY).toBeUndefined(); expect(env.TYPESAFE_API_KEY).toBeUndefined();
@@ -98,6 +103,19 @@ describe('Infisical boundary', () => {
expect(env.NODE_OPTIONS).toBeUndefined(); expect(env.NODE_OPTIONS).toBeUndefined();
expect(Object.keys(env).some(key => key.startsWith('INFISICAL_'))).toBe(false); expect(Object.keys(env).some(key => key.startsWith('INFISICAL_'))).toBe(false);
expect(applicationEnvironment(base, values, config, true)[ADMIN_KEY]).toBe('vault-admin'); expect(applicationEnvironment(base, values, config, true)[ADMIN_KEY]).toBe('vault-admin');
const workerEnv = applicationEnvironment(base, values, config, false, 'worker');
expect(workerEnv.CONVEX_SYNC_WORKER_SECRET).toBe('worker');
expect(workerEnv.CONVEX_BRIDGE_SECRET).toBeUndefined();
expect(workerEnv.APP_AUTH_PRIVATE_KEY).toBeUndefined();
expect(workerEnv.APP_AUTH_JWKS).toBeUndefined();
const configureEnv = applicationEnvironment(base, values, { ...config, APP_AUTH_ISSUER: 'https://issuer.example', APP_CAPABILITY_ALLOWLIST: 'asset_sync' }, true, 'configure');
expect(configureEnv.CONVEX_BRIDGE_SECRET).toBe('bridge');
expect(configureEnv.CONVEX_SYNC_WORKER_SECRET).toBe('worker');
expect(configureEnv.APP_AUTH_JWKS).toBe('jwks');
expect(configureEnv[ADMIN_KEY]).toBe('vault-admin');
expect(configureEnv.APP_AUTH_ISSUER).toBe('https://issuer.example');
expect(configureEnv.APP_CAPABILITY_ALLOWLIST).toBe('asset_sync');
for (const excluded of ['TOKEN_ENCRYPTION_KEY', 'APP_AUTH_PRIVATE_KEY', 'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY']) expect(configureEnv[excluded]).toBeUndefined();
expect(() => applicationEnvironment(base, values, { INFISICAL_TOKEN: 'do-not-forward' })).toThrow('Unexpected public'); expect(() => applicationEnvironment(base, values, { INFISICAL_TOKEN: 'do-not-forward' })).toThrow('Unexpected public');
expect(() => applicationEnvironment(base, values, { NODE_OPTIONS: 'do-not-execute' })).toThrow('Unexpected public'); expect(() => applicationEnvironment(base, values, { NODE_OPTIONS: 'do-not-execute' })).toThrow('Unexpected public');
}); });
+32
View File
@@ -0,0 +1,32 @@
import { afterEach, expect, test } from 'bun:test';
import { POST } from '../src/routes/api/sync/cancel/+server';
const originalEnv = { APP_MODE: process.env.APP_MODE, APP_ORIGIN: process.env.APP_ORIGIN, APP_CAPABILITY_ALLOWLIST: process.env.APP_CAPABILITY_ALLOWLIST };
afterEach(() => {
for (const [key, value] of Object.entries(originalEnv)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; }
});
function event(request: Request): Parameters<typeof POST>[0] {
return { request, url: new URL(request.url), cookies: { get: () => undefined } as any, getClientAddress: () => '127.0.0.1' } as Parameters<typeof POST>[0];
}
test('sync cancellation rejects hostile origins before session or body work', async () => {
process.env.APP_MODE = 'live'; process.env.APP_ORIGIN = 'http://localhost:5173';
process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync';
await expect(POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'https://evil.example', 'content-type': 'application/json' }, body: '{}' })))).rejects.toMatchObject({ status: 403 });
});
test('sync cancellation returns unauthenticated before reading a request body', async () => {
process.env.APP_MODE = 'live'; process.env.APP_ORIGIN = 'http://localhost:5173';
process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync';
const response = await POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'http://localhost:5173', 'content-type': 'application/json' }, body: '{}' })));
expect(response.status).toBe(401);
});
test('sync cancellation applies the bounded JSON byte cap', async () => {
process.env.APP_MODE = 'demo'; process.env.APP_ORIGIN = 'http://localhost:5173';
process.env.APP_CAPABILITY_ALLOWLIST = 'asset_sync';
const response = POST(event(new Request('http://localhost:5173/api/sync/cancel', { method: 'POST', headers: { origin: 'http://localhost:5173', 'content-type': 'application/json' }, body: JSON.stringify({ jobId: 'x'.repeat(17_000) }) })));
await expect(response).rejects.toMatchObject({ status: 413 });
});
+482
View File
@@ -0,0 +1,482 @@
import { describe, expect, test } from 'bun:test';
import { makeFunctionReference } from 'convex/server';
import { convexTest } from 'convex-test';
import schema from '../src/convex/schema';
import { EsiRateLimitError } from '../src/lib/server/esi';
import { processSyncJob } from '../workers/sync';
process.env.CONVEX_BRIDGE_SECRET = 'test-bridge-secret';
process.env.CONVEX_SYNC_WORKER_SECRET = 'test-worker-secret';
process.env.APP_CAPABILITY_ALLOWLIST = 'asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions';
const modules = {
'src/convex/functions.ts': () => import('../src/convex/functions'),
'src/convex/status.ts': () => import('../src/convex/status'),
'src/convex/_generated/server.ts': async () => ({}),
};
const mutation = (name: string) => makeFunctionReference<'mutation', any, any>(`functions:${name}`);
const query = (name: string) => makeFunctionReference<'query', any, any>(`functions:${name}`);
const bridge = { bridgeSecret: 'test-bridge-secret' };
const worker = { workerSecret: 'test-worker-secret' };
async function login(t: ReturnType<typeof convexTest>, input: Record<string, unknown> = {}) {
return await t.mutation(mutation('bridgeCompleteLogin'), {
...bridge, characterId: 700, name: 'Queue Pilot', ownerHash: 'queue-owner', scopes: ['esi-assets.read_assets.v1'],
intent: 'initial_login', capability: 'asset_search', requestedScopes: ['esi-assets.read_assets.v1'], approvedExistingScopes: [],
encryptedTokens: 'encrypted-token', tokenExpiresAt: Date.now() + 60_000, sessionHash: 'queue-session', sessionExpiresAt: Date.now() + 60_000,
...input,
});
}
async function enqueue(t: ReturnType<typeof convexTest>, account: any, characterId = 700, sessionHash = 'queue-session') {
return await t.mutation(mutation('bridgeEnqueueAssetSync'), { ...bridge, userId: account.userId, characterId, initiatingSessionHash: sessionHash });
}
async function cloneCharacters(t: ReturnType<typeof convexTest>, userId: string, sourceCharacterId: number, characterIds: number[]) {
await t.run(async (ctx: any) => {
const source = await ctx.db.query('characters').withIndex('by_user_character', (q: any) => q.eq('userId', userId).eq('characterId', sourceCharacterId)).first();
const token = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', userId).eq('characterId', sourceCharacterId)).first();
const { _id: _characterId, _creationTime: _characterCreated, ...characterFields } = source;
const { _id: _tokenId, _creationTime: _tokenCreated, ...tokenFields } = token;
for (const characterId of characterIds) {
await ctx.db.insert('characters', { ...characterFields, characterId, name: `Clone ${characterId}` });
await ctx.db.insert('tokenRecords', { ...tokenFields, characterId });
}
});
}
describe('T07 durable asset scheduling', () => {
test('derives authority, isolates tenants, and deduplicates one active job', async () => {
const t = convexTest({ schema, modules });
const first = await login(t);
const duplicate = await enqueue(t, first);
const same = await enqueue(t, first);
expect(same.id).toBe(duplicate.id);
const other = await login(t, { characterId: 701, sessionHash: 'other-session', ownerHash: 'other-owner' });
const otherJob = await enqueue(t, other, 701, 'other-session');
expect(otherJob.id).not.toBe(duplicate.id);
await expect(t.query(query('bridgeSyncJob'), { ...bridge, userId: other.userId, jobId: duplicate.id })).resolves.toBeNull();
await expect(t.mutation(mutation('bridgeEnqueueAssetSync'), { ...bridge, userId: other.userId, characterId: 700, initiatingSessionHash: 'other-session' })).rejects.toThrow('active');
});
test('claims only user asset jobs, reclaims an expired lease, and keeps unsupported jobs untouched', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 708, sessionHash: 'claim-filter-session' });
const job = await enqueue(t, account, 708, 'claim-filter-session');
await t.run(async (ctx: any) => {
await ctx.db.insert('jobs', { userId: account.userId, kind: 'asset_sync', dataset: 'assets', authority: 'system', ownerKind: 'system', status: 'queued', createdAt: Date.now(), updatedAt: Date.now(), dueAt: Date.now(), attempts: 0, maxAttempts: 5, priority: 0 });
await ctx.db.insert('jobs', { userId: account.userId, kind: 'reference_sync', dataset: 'reference', authority: 'system', ownerKind: 'system', status: 'queued', createdAt: Date.now(), updatedAt: Date.now(), dueAt: Date.now(), attempts: 0, maxAttempts: 5, priority: 0 });
});
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-filter', leaseMs: 6 * 60_000 });
expect(claimed.jobId).toBe(job.id);
await t.run(async (ctx: any) => {
const row = await ctx.db.get(claimed.jobId);
await ctx.db.patch(row._id, { leaseExpiresAt: 0 });
});
const reclaimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-reclaim', leaseMs: 6 * 60_000 });
expect(reclaimed.jobId).toBe(job.id);
expect(reclaimed.attempts).toBe(2);
});
test('claims, renews beyond the former five-minute lease, and cancellation fences delivery', async () => {
const t = convexTest({ schema, modules });
const account = await login(t);
const job = await enqueue(t, account);
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-a', leaseMs: 6 * 60_000 });
expect(claimed).toMatchObject({ jobId: job.id, status: 'running', attempts: 1 });
await t.run(async (ctx: any) => await ctx.db.patch(claimed.jobId, { leaseExpiresAt: Date.now() + 301_000 }));
await expect(t.mutation(mutation('bridgeRenewSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, workerId: 'worker-a', leaseMs: 6 * 60_000 })).resolves.toBe(true);
await expect(t.mutation(mutation('bridgeCancelSyncJob'), { ...bridge, userId: account.userId, jobId: claimed.jobId, initiatingSessionHash: 'queue-session' })).resolves.toMatchObject({ status: 'cancelled' });
await expect(t.mutation(mutation('bridgeRenewSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, workerId: 'worker-a', leaseMs: 6 * 60_000 })).resolves.toBe(false);
});
test('enforces the per-user queue cap and fair claims across users', async () => {
const t = convexTest({ schema, modules });
const first = await login(t, { characterId: 709, sessionHash: 'cap-session' });
await cloneCharacters(t, first.userId, 709, [710, 711, 712, 713, 714, 715, 716, 717, 718]);
for (const characterId of [709, 710, 711, 712, 713, 714, 715, 716]) await enqueue(t, first, characterId, 'cap-session');
await expect(enqueue(t, first, 717, 'cap-session')).rejects.toThrow('queue is full');
const second = await login(t, { characterId: 719, sessionHash: 'fair-second-session', ownerHash: 'fair-second-owner' });
await enqueue(t, second, 719, 'fair-second-session');
const claimedFirst = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-fair-a' });
const claimedSecond = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-fair-b' });
expect(claimedFirst.userId).toBe(first.userId);
expect(claimedSecond.userId).toBe(second.userId);
});
test('shared provider cooldown blocks a different character until its eligible time', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 720, sessionHash: 'cooldown-session' });
await cloneCharacters(t, account.userId, 720, [721]);
const first = await enqueue(t, account, 720, 'cooldown-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-a' });
const until = Date.now() + 30_000;
await t.mutation(mutation('bridgeDeferSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, dueAt: until, cooldownContext: `character:${account.userId}:720`, cooldownUntil: until, cooldownGroup: 'esi:assets' });
await enqueue(t, account, 721, 'cooldown-session');
await expect(t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-b' })).resolves.toBeNull();
await t.run(async (ctx: any) => {
const cooldown = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', 'group:esi:assets')).first();
if (cooldown) await ctx.db.patch(cooldown._id, { until: 0 });
const jobs = await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', account.userId).eq('status', 'queued')).collect();
for (const job of jobs) await ctx.db.patch(job._id, { dueAt: 0 });
});
await expect(t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cooldown-c' })).resolves.toMatchObject({ userId: account.userId });
expect(first.status).toBe('queued');
});
test('stages idempotently and fences stale success and failure after credential rotation', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 703, sessionHash: 'publish-session' });
const queued = await enqueue(t, account, 703, 'publish-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-publish' });
const generation = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken });
const row = { itemId: '9007199254740993', characterId: 703, typeId: 34, quantity: 1, locationId: '60003760', locationType: 'station', locationFlag: 'Hangar', isSingleton: true };
await t.mutation(mutation('bridgeStageSyncAssets'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, index: 0, assets: [row] });
await t.mutation(mutation('bridgeStageSyncAssets'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, index: 0, assets: [row] });
await t.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: account.userId, characterId: 703, encryptedTokens: 'rotated', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'], expectedVersion: 1 });
const rotatedCharacters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
const rotated = rotatedCharacters.find((entry: any) => entry.characterId === 703);
expect(rotated.syncGeneration).toBeUndefined();
expect(rotated.datasetStatuses.find((entry: any) => entry.dataset === 'assets')).toMatchObject({ status: 'blocked' });
await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 1, syncedAt: Date.now(), expiresAt: Date.now() + 60_000 })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow();
const page = await t.query(query('bridgeAssetsPage'), { ...bridge, userId: account.userId, characterId: 703, paginationOpts: { numItems: 10, cursor: null } });
expect(page.generation).toBeNull();
expect(generation).toEqual(expect.any(String));
expect(queued.status).toBe('queued');
});
test('duplicate commit delivery is idempotent and cancellation fences stale success and failure', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 722, sessionHash: 'duplicate-session' });
await enqueue(t, account, 722, 'duplicate-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-duplicate' });
await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken });
await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 });
await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).resolves.toBe(true);
const second = await login(t, { characterId: 723, sessionHash: 'cancel-fence-session' });
await enqueue(t, second, 723, 'cancel-fence-session');
const cancelled = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-cancel-fence' });
await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken });
await t.mutation(mutation('bridgeCancelSyncJob'), { ...bridge, userId: second.userId, jobId: cancelled.jobId, initiatingSessionHash: 'cancel-fence-session' });
await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: cancelled.jobId, leaseToken: cancelled.leaseToken })).rejects.toThrow();
});
test('authorization quarantine fences both stale success and stale failure', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 727, sessionHash: 'quarantine-session' });
await enqueue(t, account, 727, 'quarantine-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-quarantine' });
await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken });
await t.mutation(mutation('bridgeQuarantineAuthorization'), { ...bridge, userId: account.userId, characterId: 727 });
await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow();
});
test('worker retries applied-then-lost staging and commit acknowledgements idempotently', async () => {
const events: string[] = [];
let stageCalls = 0; let commitCalls = 0;
const claimed = { id: 'job-lost', jobId: 'job-lost', userId: 'user-lost', characterId: 728, leaseToken: 'lease-lost', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any;
const store = {
claimSyncJob: async () => claimed,
renewSyncJob: async () => { events.push('renew'); return true; },
acquireRefreshLease: async () => ({ leaseToken: 'refresh-lost', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }),
beginSyncJob: async () => 'generation-lost',
cooldown: async () => null,
stageSyncAssets: async () => { stageCalls += 1; events.push(`stage${stageCalls}`); if (stageCalls === 1) throw new Error('response lost after stage'); },
commitSyncJob: async () => { commitCalls += 1; events.push(`commit${commitCalls}`); if (commitCalls === 1) throw new Error('response lost after commit'); return true; },
failSyncJob: async () => { events.push('fail'); return true; },
deferSyncJob: async () => { events.push('defer'); return true; },
};
await processSyncJob({
store: store as any,
cipher: { open: async () => ({ characterId: 728, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any,
sso: {} as any,
esi: { getAssets: async () => ({ assets: [{ itemId: '1', characterId: 728, typeId: 34, quantity: 1, locationId: '2', locationType: 'station', locationFlag: 'Hangar' }], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any,
workerId: 'worker-lost', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {},
});
expect(stageCalls).toBe(2);
expect(commitCalls).toBe(2);
expect(events).toEqual(['stage1', 'renew', 'stage2', 'commit1', 'renew', 'commit2']);
});
test('ambiguous begin and commit mutations defer for lease recovery instead of failing terminally', async () => {
const beginEvents: string[] = [];
const beginStore = {
claimSyncJob: async () => ({ id: 'job-begin-uncertain', jobId: 'job-begin-uncertain', userId: 'user-begin', characterId: 740, leaseToken: 'lease-begin', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any),
renewSyncJob: async () => { beginEvents.push('renew'); return true; },
acquireRefreshLease: async () => ({ leaseToken: 'refresh-begin', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }),
beginSyncJob: async () => { beginEvents.push('begin'); throw new Error('response lost after begin'); },
deferSyncJob: async () => { beginEvents.push('defer'); return true; },
failSyncJob: async () => { beginEvents.push('fail'); return true; },
};
await processSyncJob({ store: beginStore as any, cipher: { open: async () => ({ characterId: 740, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, sso: {} as any, esi: {} as any, workerId: 'worker-begin', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} });
expect(beginEvents).toEqual(['begin', 'renew', 'begin', 'defer']);
const commitEvents: string[] = [];
const commitStore = {
claimSyncJob: async () => ({ id: 'job-commit-uncertain', jobId: 'job-commit-uncertain', userId: 'user-commit', characterId: 741, leaseToken: 'lease-commit', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any),
renewSyncJob: async () => { commitEvents.push('renew'); return true; },
acquireRefreshLease: async () => ({ leaseToken: 'refresh-commit', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }),
beginSyncJob: async () => 'generation-commit-uncertain',
cooldown: async () => null,
stageSyncAssets: async () => { commitEvents.push('stage'); },
commitSyncJob: async () => { commitEvents.push('commit'); throw new Error('response lost after commit'); },
deferSyncJob: async () => { commitEvents.push('defer'); return true; },
failSyncJob: async () => { commitEvents.push('fail'); return true; },
};
await processSyncJob({ store: commitStore as any, cipher: { open: async () => ({ characterId: 741, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any, sso: {} as any, esi: { getAssets: async () => ({ assets: [], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any, workerId: 'worker-commit', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} });
expect(commitEvents).toEqual(['commit', 'renew', 'commit', 'defer']);
});
test('ambiguous restart after refresh commit defers without replaying the remote refresh', async () => {
const events: string[] = [];
const store = {
claimSyncJob: async () => ({ id: 'job-restart-uncertain', jobId: 'job-restart-uncertain', userId: 'user-restart', characterId: 742, leaseToken: 'lease-restart', status: 'running', attempts: 1, generation: 'old-generation', dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any),
renewSyncJob: async () => { events.push('renew'); return true; },
acquireRefreshLease: async () => ({ leaseToken: 'refresh-restart', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
markRefreshDispatched: async () => { events.push('dispatch'); return true; },
commitRefresh: async () => { events.push('refresh-commit'); return { credentialVersion: 2, grantRevision: 1, authorizationEpoch: 1, scopes: ['esi-assets.read_assets.v1'] }; },
restartSyncJobGeneration: async () => { events.push('restart'); throw new Error('response lost after restart'); },
markRefreshUncertain: async () => { events.push('uncertain'); return true; },
deferSyncJob: async () => { events.push('defer'); return true; },
failSyncJob: async () => { events.push('fail'); return true; },
};
await processSyncJob({ store: store as any, cipher: { open: async () => ({ characterId: 742, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'] }), seal: async () => 'rotated-sealed' } as any, sso: { refresh: async () => { events.push('remote'); return { characterId: 742, ownerHash: 'owner', accessToken: 'new-access', refreshToken: 'new-refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }; } } as any, esi: {} as any, workerId: 'worker-restart', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {} });
expect(events).toEqual(['dispatch', 'remote', 'refresh-commit', 'restart', 'renew', 'restart', 'uncertain', 'defer']);
});
test('worker publishes a complete empty snapshot through the leased lifecycle', async () => {
const events: string[] = [];
const claimed = { id: 'job-1', jobId: 'job-1', userId: 'user-1', characterId: 700, leaseToken: 'lease-1', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any;
const store = {
claimSyncJob: async () => claimed,
renewSyncJob: async () => true,
acquireRefreshLease: async () => ({ leaseToken: 'refresh-1', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
releaseRefreshLease: async () => { events.push('release'); return true; },
refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }),
beginSyncJob: async () => { events.push('begin'); return 'generation-1'; },
cooldown: async () => null,
stageSyncAssets: async () => { events.push('stage'); },
commitSyncJob: async () => { events.push('commit'); return true; },
failSyncJob: async () => { events.push('fail'); return true; },
deferSyncJob: async () => { events.push('defer'); return true; },
};
const handled = await processSyncJob({
store: store as any,
cipher: { open: async () => ({ characterId: 700, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any,
sso: { refresh: async () => { throw new Error('refresh should not run'); } } as any,
esi: { getAssets: async () => ({ assets: [], syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 }) } as any,
workerId: 'worker-1', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {},
});
expect(handled).toBe(true);
expect(events).toEqual(['release', 'begin', 'commit']);
});
test('worker durably marks refresh dispatch before a failed remote attempt', async () => {
const events: string[] = [];
const claimed = { id: 'job-refresh', jobId: 'job-refresh', userId: 'user-refresh', characterId: 725, leaseToken: 'lease-refresh', status: 'running', attempts: 1, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any;
const store = {
claimSyncJob: async () => claimed,
renewSyncJob: async () => true,
acquireRefreshLease: async () => ({ leaseToken: 'refresh-dispatch', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
markRefreshDispatched: async () => { events.push('dispatch'); return true; },
markRefreshUncertain: async () => { events.push('uncertain'); return true; },
failSyncJob: async () => { events.push('fail'); return true; },
deferSyncJob: async () => { events.push('defer'); return true; },
};
await processSyncJob({
store: store as any,
cipher: { open: async () => ({ characterId: 725, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 1_000, scopes: ['esi-assets.read_assets.v1'] }) } as any,
sso: { refresh: async () => { events.push('remote'); throw new Error('unknown remote outcome'); } } as any,
esi: {} as any,
workerId: 'worker-refresh', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {},
});
expect(events).toEqual(['dispatch', 'remote', 'uncertain']);
});
test('worker defers rate limits into the durable cooldown instead of failing the job', async () => {
const events: Array<{ name: string; dueAt?: number; cooldown?: unknown }> = [];
const claimed = { id: 'job-2', jobId: 'job-2', userId: 'user-2', characterId: 701, leaseToken: 'lease-2', status: 'running', attempts: 2, dueAt: Date.now(), createdAt: Date.now(), updatedAt: Date.now(), grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] } as any;
const retryAt = Date.now() + 15_000;
const store = {
claimSyncJob: async () => claimed,
renewSyncJob: async () => true,
acquireRefreshLease: async () => ({ leaseToken: 'refresh-2', expiresAt: Date.now() + 120_000, expectedVersion: 1, encryptedTokens: 'sealed', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1 }),
refreshSyncJobAuthority: async () => ({ grantRevision: 1, authorizationEpoch: 1, credentialVersion: 1, scopes: ['esi-assets.read_assets.v1'] }),
beginSyncJob: async () => 'generation-2',
cooldown: async () => null,
stageSyncAssets: async () => {},
commitSyncJob: async () => true,
failSyncJob: async () => { events.push({ name: 'fail' }); return true; },
deferSyncJob: async (_jobId: string, _lease: string, dueAt: number, cooldown?: unknown) => { events.push({ name: 'defer', dueAt, cooldown }); return true; },
};
await processSyncJob({
store: store as any,
cipher: { open: async () => ({ characterId: 701, ownerHash: 'owner', accessToken: 'access', refreshToken: 'refresh', expiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'] }) } as any,
sso: { refresh: async () => { throw new Error('refresh should not run'); } } as any,
esi: { getAssets: async () => { throw new EsiRateLimitError('/latest/characters/701/assets', retryAt, 15_000, 'esi:public'); } } as any,
workerId: 'worker-2', pollMs: 1, leaseMs: 600_000, heartbeatMs: 60_000, sleep: async () => {},
});
expect(events).toHaveLength(1);
expect(events[0]?.name).toBe('defer');
expect(events[0]?.dueAt).toBe(retryAt);
expect(events[0]?.cooldown).toMatchObject({ context: 'character:user-2:701', until: retryAt });
});
test('terminal retry exhaustion clears syncing state while preserving the prior generation', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 724, sessionHash: 'terminal-session' });
await enqueue(t, account, 724, 'terminal-session');
const first = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-terminal-first' });
await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: first.jobId, leaseToken: first.leaseToken });
await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: first.jobId, leaseToken: first.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 });
await enqueue(t, account, 724, 'terminal-session');
const second = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-terminal-second' });
await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: second.jobId, leaseToken: second.leaseToken });
await t.run(async (ctx: any) => await ctx.db.patch(second.jobId, { attempts: 5 }));
await expect(t.mutation(mutation('bridgeDeferSyncJob'), { ...worker, jobId: second.jobId, leaseToken: second.leaseToken, dueAt: Date.now() + 1_000 })).resolves.toBe(false);
const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
const character = characters.find((entry: any) => entry.characterId === 724);
expect(character).toMatchObject({ state: 'error', activeGeneration: expect.any(String) });
expect(character.syncGeneration).toBeUndefined();
});
test('reclaims a lease after refresh commit before generation restart and publishes a fresh generation', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 703, sessionHash: 'refresh-crash-session' });
const queued = await enqueue(t, account, 703, 'refresh-crash-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-refresh-crash' });
const oldGeneration = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken });
const refresh = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash' });
await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash', leaseToken: refresh.leaseToken });
await t.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: account.userId, characterId: 703, ownerId: 'worker-refresh-crash', leaseToken: refresh.leaseToken, expectedVersion: refresh.expectedVersion, encryptedTokens: 'rotated-token', tokenExpiresAt: Date.now() + 120_000, scopes: ['esi-assets.read_assets.v1'], jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken });
await t.run(async (ctx: any) => await ctx.db.patch(queued.id, { leaseExpiresAt: 0 }));
const reclaimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-refresh-reclaimed' });
const freshGeneration = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: reclaimed.jobId, leaseToken: reclaimed.leaseToken });
expect(freshGeneration).not.toBe(oldGeneration);
await t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: reclaimed.jobId, leaseToken: reclaimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 });
const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
expect(characters.find((entry: any) => entry.characterId === 703)).toMatchObject({ state: 'ready', activeGeneration: freshGeneration });
});
test('refresh commit validates allowlisted subset scopes and fences reduced grants', async () => {
const makeLease = async (t: ReturnType<typeof convexTest>, account: any, ownerId: string) => {
const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: account.characterId, ownerId });
await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: account.characterId, ownerId, leaseToken: lease.leaseToken });
return lease;
};
const invalid = convexTest({ schema, modules });
const invalidAccount = await login(invalid, { characterId: 730, sessionHash: 'scope-invalid-session' });
const invalidLease = await makeLease(invalid, { ...invalidAccount, characterId: 730 }, 'scope-invalid');
await expect(invalid.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: invalidAccount.userId, characterId: 730, ownerId: 'scope-invalid', leaseToken: invalidLease.leaseToken, expectedVersion: invalidLease.expectedVersion, encryptedTokens: 'bad', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-unknown.scope'] })).rejects.toThrow('Invalid refreshed scopes');
await expect(invalid.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: invalidAccount.userId, characterId: 730, encryptedTokens: 'expanded', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1', 'esi-wallet.read_character_wallet.v1'], expectedVersion: 1 })).resolves.toBe(false);
const expanded = convexTest({ schema, modules });
const expandedAccount = await login(expanded, { characterId: 731, sessionHash: 'scope-expanded-session' });
const expandedLease = await makeLease(expanded, { ...expandedAccount, characterId: 731 }, 'scope-expanded');
await expect(expanded.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: expandedAccount.userId, characterId: 731, ownerId: 'scope-expanded', leaseToken: expandedLease.leaseToken, expectedVersion: expandedLease.expectedVersion, encryptedTokens: 'bad', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1', 'esi-wallet.read_character_wallet.v1'] })).rejects.toThrow('Invalid refreshed scopes');
const reduced = convexTest({ schema, modules });
const reducedAccount = await login(reduced, { characterId: 732, sessionHash: 'scope-reduced-session' });
const reducedJob = await enqueue(reduced, reducedAccount, 732, 'scope-reduced-session');
const reducedClaim = await reduced.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'scope-reduced-worker' });
await reduced.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: reducedClaim.jobId, leaseToken: reducedClaim.leaseToken });
await reduced.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: reducedClaim.jobId, leaseToken: reducedClaim.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 });
const reducedLease = await makeLease(reduced, { ...reducedAccount, characterId: 732 }, 'scope-reduced');
await expect(reduced.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: reducedAccount.userId, characterId: 732, ownerId: 'scope-reduced', leaseToken: reducedLease.leaseToken, expectedVersion: reducedLease.expectedVersion, encryptedTokens: 'reduced', tokenExpiresAt: Date.now() + 60_000, scopes: [] })).resolves.toMatchObject({ saved: true });
const reducedCharacter = (await reduced.query(query('bridgeCharacters'), { ...bridge, userId: reducedAccount.userId })).find((entry: any) => entry.characterId === 732);
expect(reducedCharacter).toMatchObject({ state: 'error' });
expect(reducedCharacter.activeGeneration).toBeUndefined();
const unchanged = convexTest({ schema, modules });
const unchangedAccount = await login(unchanged, { characterId: 733, sessionHash: 'scope-unchanged-session' });
const unchangedLease = await makeLease(unchanged, { ...unchangedAccount, characterId: 733 }, 'scope-unchanged');
await expect(unchanged.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: unchangedAccount.userId, characterId: 733, ownerId: 'scope-unchanged', leaseToken: unchangedLease.leaseToken, expectedVersion: unchangedLease.expectedVersion, encryptedTokens: 'unchanged', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'] })).resolves.toMatchObject({ saved: true, grantRevision: 1, credentialVersion: 2 });
});
test('first-generation asset scope loss quarantines staging and clears the matching pointer', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 736, sessionHash: 'first-scope-loss-session' });
await enqueue(t, account, 736, 'first-scope-loss-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'first-scope-loss-worker' });
const generation = await t.mutation(mutation('bridgeBeginSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken });
const refresh = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker' });
await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker', leaseToken: refresh.leaseToken });
await t.mutation(mutation('bridgeCommitRefresh'), { ...worker, userId: account.userId, characterId: 736, ownerId: 'first-scope-loss-worker', leaseToken: refresh.leaseToken, expectedVersion: refresh.expectedVersion, encryptedTokens: 'reduced-first', tokenExpiresAt: Date.now() + 60_000, scopes: [], jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken });
const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
const character = characters.find((entry: any) => entry.characterId === 736);
expect(character.state).toBe('error');
expect(character.syncGeneration).toBeUndefined();
expect(character.syncLeaseExpiresAt).toBeUndefined();
expect(character.datasetStatuses.find((entry: any) => entry.dataset === 'assets')).toMatchObject({ status: 'blocked' });
const job = await t.query(query('bridgeSyncJob'), { ...bridge, userId: account.userId, jobId: claimed.jobId });
expect(job).toMatchObject({ status: 'quarantined' });
await expect(t.mutation(mutation('bridgeCommitSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, chunkCount: 0, syncedAt: Date.now() - 1_000, expiresAt: Date.now() + 60_000 })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeFailSyncJob'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken })).rejects.toThrow();
await t.run(async (ctx: any) => {
const row = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', account.userId).eq('characterId', 736).eq('generation', generation)).first();
expect(row.status).toBe('failed');
});
});
test('refresh release is fenced and lets a subsequent worker acquire immediately', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 734, sessionHash: 'release-session' });
const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner' });
await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'foreign-owner', leaseToken: lease.leaseToken })).resolves.toBe(false);
await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner', leaseToken: lease.leaseToken })).resolves.toBe(true);
await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'release-owner', leaseToken: lease.leaseToken })).resolves.toBe(false);
const next = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner' });
expect(typeof next.leaseToken).toBe('string');
expect(next.expectedVersion).toBe(1);
await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner', leaseToken: next.leaseToken });
await expect(t.mutation(mutation('bridgeReleaseRefreshLease'), { ...worker, userId: account.userId, characterId: 734, ownerId: 'next-owner', leaseToken: next.leaseToken })).resolves.toBe(false);
});
test('authority refresh rejects a same-scope credential rotation instead of adopting newer DB counters', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 704, sessionHash: 'stale-authority-session' });
const job = await enqueue(t, account, 704, 'stale-authority-session');
const claimed = await t.mutation(mutation('bridgeClaimSyncJob'), { ...worker, workerId: 'worker-stale' });
await t.mutation(mutation('bridgeSaveTokens'), { ...bridge, userId: account.userId, characterId: 704, encryptedTokens: 'new-token', tokenExpiresAt: Date.now() + 60_000, scopes: ['esi-assets.read_assets.v1'], expectedVersion: 1 });
await expect(t.mutation(mutation('bridgeRefreshSyncJobAuthority'), { ...worker, jobId: claimed.jobId, leaseToken: claimed.leaseToken, expectedGrantRevision: 1, expectedAuthorizationEpoch: 1, expectedCredentialVersion: 1, expectedScopes: ['esi-assets.read_assets.v1'] })).rejects.toThrow();
expect(job.status).toBe('queued');
});
test('expired dispatched refresh leases fail closed without replaying a rotated token', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 705, sessionHash: 'dispatch-session' });
const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-dispatch' });
await t.mutation(mutation('bridgeMarkRefreshDispatched'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-dispatch', leaseToken: lease.leaseToken });
await t.run(async (ctx: any) => {
const row = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', account.userId).eq('characterId', 705)).first();
await ctx.db.patch(row._id, { expiresAt: 0 });
});
await expect(t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 705, ownerId: 'worker-replay' })).resolves.toBeNull();
const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
expect(characters.find((character: any) => character.characterId === 705)).toMatchObject({ authorizationStatus: 'reconnect_required' });
await expect(t.query(query('bridgeTokenRecord'), { ...bridge, userId: account.userId, characterId: 705 })).resolves.toBeNull();
});
test('worker credentials cannot invoke general login, enqueue, or cancellation bridges', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 706, sessionHash: 'least-privilege-session' });
await expect(t.mutation(mutation('bridgeCompleteLogin'), { ...worker, characterId: 707, name: 'Nope', ownerHash: 'nope', scopes: ['esi-assets.read_assets.v1'], intent: 'initial_login', capability: 'asset_search', requestedScopes: ['esi-assets.read_assets.v1'], approvedExistingScopes: [], encryptedTokens: 'x', tokenExpiresAt: Date.now() + 60_000, sessionHash: 'bad-worker-session', sessionExpiresAt: Date.now() + 60_000 })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeEnqueueAssetSync'), { ...worker, userId: account.userId, characterId: 706, initiatingSessionHash: 'least-privilege-session' })).rejects.toThrow();
await expect(t.mutation(mutation('bridgeCancelSyncJob'), { ...worker, userId: account.userId, jobId: 'missing', initiatingSessionHash: 'least-privilege-session' })).rejects.toThrow();
});
test('refresh lease is durable, CAS committed, and uncertain rotation fails closed', async () => {
const t = convexTest({ schema, modules });
const account = await login(t, { characterId: 702, sessionHash: 'refresh-session' });
const lease = await t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-a' });
expect(lease).toMatchObject({ expectedVersion: 1 });
await expect(t.mutation(mutation('bridgeAcquireRefreshLease'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-b' })).resolves.toBeNull();
await expect(t.mutation(mutation('bridgeMarkRefreshUncertain'), { ...worker, userId: account.userId, characterId: 702, ownerId: 'worker-a', leaseToken: lease.leaseToken, expectedVersion: 1 })).resolves.toBe(true);
const characters = await t.query(query('bridgeCharacters'), { ...bridge, userId: account.userId });
expect(characters[0]).toMatchObject({ authorizationStatus: 'reconnect_required' });
await expect(t.query(query('bridgeTokenRecord'), { ...bridge, userId: account.userId, characterId: 702 })).resolves.toBeNull();
});
});
+75
View File
@@ -0,0 +1,75 @@
import { describe, expect, test } from 'bun:test';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { parseSyncEnqueueBody } from '../src/lib/client/sync-queue';
import { workerEnvironment } from '../scripts/run-sync-worker';
describe('T07 worker runtime boundary', () => {
test('bundled worker starts with inert restricted configuration and has no unresolved aliases', async () => {
const directory = await mkdtemp(join(tmpdir(), 'eve-sync-worker-'));
try {
const output = join(directory, 'sync-worker.js');
const built = await Bun.build({ entrypoints: ['workers/sync.ts'], target: 'bun', outdir: directory, naming: 'sync-worker.js', minify: false });
expect(built.success).toBe(true);
const source = await readFile(output, 'utf8');
expect(source.includes('$lib/')).toBe(false);
const env: Record<string, string | undefined> = { PATH: process.env.PATH, HOME: process.env.HOME, CONVEX_URL: 'http://127.0.0.1:9', CONVEX_SYNC_WORKER_SECRET: 'inert-worker-secret', TOKEN_ENCRYPTION_KEY: Buffer.alloc(32).toString('base64'), EVE_CLIENT_ID: 'inert-client', EVE_CALLBACK_URL: 'http://localhost/unused-worker-callback', ESI_USER_AGENT: 'EVE-Intelligence/test', SYNC_WORKER_STARTUP_CHECK: '1' };
for (const name of ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY']) delete env[name];
const child = Bun.spawn(['bun', output], { env, stdout: 'pipe', stderr: 'pipe' });
const code = await child.exited;
const outputText = await new Response(child.stdout).text() + await new Response(child.stderr).text();
expect(code).toBe(0);
expect(outputText).toBe('');
} finally { await rm(directory, { recursive: true, force: true }); }
});
test('client preserves structured all-skipped and partial enqueue outcomes on HTTP errors', async () => {
expect(parseSyncEnqueueBody({ queued: 0, jobs: [], skipped: [{ characterId: 1, reason: 'queue_full' }], partial: true })).toMatchObject({ queued: 0, skipped: [{ characterId: 1, reason: 'queue_full' }] });
expect(parseSyncEnqueueBody({ queued: 1, jobs: [{ id: 'job', characterId: 2, status: 'queued', attempts: 0, dueAt: 1, createdAt: 1, updatedAt: 1, leaseExpiresAt: null, cancelRequested: false }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }], partial: true })).toMatchObject({ queued: 1, jobs: [{ id: 'job', characterId: 2 }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }] });
const originalFetch = globalThis.fetch;
try {
globalThis.fetch = (async () => Response.json({ queued: 0, jobs: [], skipped: [{ characterId: 1, reason: 'queue_full' }], partial: true }, { status: 429 })) as unknown as typeof fetch;
const allSkipped = await (await import('../src/lib/client/sync-queue')).enqueueSync();
expect(allSkipped.response.status).toBe(429); expect(allSkipped.body).toMatchObject({ jobs: [], skipped: [{ reason: 'queue_full' }] });
globalThis.fetch = (async () => Response.json({ queued: 1, jobs: [{ id: 'job', characterId: 2, status: 'queued', attempts: 0, dueAt: 1, createdAt: 1, updatedAt: 1, leaseExpiresAt: null, cancelRequested: false }], skipped: [{ characterId: 3, reason: 'enqueue_failed' }], partial: true }, { status: 503 })) as unknown as typeof fetch;
const partial = await (await import('../src/lib/client/sync-queue')).enqueueSync();
expect(partial.response.status).toBe(503); expect(partial.body).toMatchObject({ jobs: [{ id: 'job' }], skipped: [{ reason: 'enqueue_failed' }] });
} finally { globalThis.fetch = originalFetch; }
});
test('missing status is surfaced and status polling forwards cancellation', async () => {
const originalFetch = globalThis.fetch;
try {
globalThis.fetch = (async () => new Response(null, { status: 404 })) as unknown as typeof fetch;
expect(await (await import('../src/lib/client/sync-queue')).syncJobStatus('missing')).toBeNull();
globalThis.fetch = (() => new Promise<Response>(() => {})) as unknown as typeof fetch;
await expect((await import('../src/lib/client/sync-queue')).syncJobStatus('never', undefined, 5)).rejects.toThrow('timed out');
const controller = new AbortController();
globalThis.fetch = ((_input: RequestInfo | URL, init?: RequestInit) => new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')), { once: true });
})) as unknown as typeof fetch;
const pending = (await import('../src/lib/client/sync-queue')).syncJobStatus('running', controller.signal);
controller.abort();
await expect(pending).rejects.toMatchObject({ name: 'AbortError' });
} finally { globalThis.fetch = originalFetch; }
});
test('worker child environment drops Infisical bootstrap credentials and ambient web keys', () => {
const child = workerEnvironment({ PATH: '/bin', CONVEX_URL: 'http://inert', CONVEX_SYNC_WORKER_SECRET: 'worker-only', TOKEN_ENCRYPTION_KEY: 'inert-key', EVE_CLIENT_ID: 'inert-client', EVE_CLIENT_SECRET: 'inert-eve-secret', INFISICAL_TOKEN: 'inert-token', INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: 'inert-client-id', INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: 'inert-client-secret', CONVEX_BRIDGE_SECRET: 'inert-bridge', APP_AUTH_PRIVATE_KEY: 'inert-jwt', TYPESAFE_API_KEY: 'inert-jev' });
expect(child).toEqual({ PATH: '/bin', CONVEX_URL: 'http://inert', CONVEX_SYNC_WORKER_SECRET: 'worker-only', TOKEN_ENCRYPTION_KEY: 'inert-key', EVE_CLIENT_ID: 'inert-client', EVE_CLIENT_SECRET: 'inert-eve-secret' });
expect(Object.keys(child).some((key) => key.startsWith('INFISICAL_'))).toBe(false);
});
test('direct worker launcher refuses general bridge/admin/JWT/JEV credentials', async () => {
for (const name of ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY']) {
const forbidden = `inert-${name}`;
const env = { PATH: process.env.PATH, HOME: process.env.HOME, CONVEX_SYNC_WORKER_SECRET: 'inert-worker-secret', [name]: forbidden };
const child = Bun.spawn(['bun', 'scripts/run-sync-worker.ts'], { env, stdout: 'pipe', stderr: 'pipe' });
const code = await child.exited;
const output = await new Response(child.stdout).text() + await new Response(child.stderr).text();
expect(code).not.toBe(0);
expect(output).not.toContain(forbidden);
}
});
});
+2
View File
@@ -35,6 +35,8 @@ test('Authenticated route limits remain user-keyed after rollout gates', () => {
}); });
test('Compose and Docker defaults are isolated and digest-pinned', () => { test('Compose and Docker defaults are isolated and digest-pinned', () => {
expect(compose).toMatch(/sync-worker:\n\s+profiles: \["worker"\]/);
expect(compose).toContain('CONVEX_SYNC_WORKER_SECRET: ${CONVEX_SYNC_WORKER_SECRET:-}');
expect(compose).toContain('CONVEX_BIND_ADDRESS:-127.0.0.1'); expect(compose).toContain('CONVEX_BIND_ADDRESS:-127.0.0.1');
expect(compose).toContain('WEB_BIND_ADDRESS:-127.0.0.1'); expect(compose).toContain('WEB_BIND_ADDRESS:-127.0.0.1');
expect(compose).toMatch(/CONVEX_IMAGE:-[^\n]*@sha256:[a-f0-9]{64}/); expect(compose).toMatch(/CONVEX_IMAGE:-[^\n]*@sha256:[a-f0-9]{64}/);
+222
View File
@@ -0,0 +1,222 @@
import { ConvexStore, type RefreshLease } from '../src/lib/server/store';
import { TokenCipher } from '../src/lib/server/security';
import { EveSso, type VerifiedGrant } from '../src/lib/server/sso';
import { EsiClient, EsiError, EsiHttpError, EsiRateLimitError } from '../src/lib/server/esi';
/** Conservative prototype budgets; these are intentionally configurable and bounded. */
export const SYNC_WORKER_DEFAULTS = {
pollMs: 1_000,
leaseMs: 10 * 60_000,
heartbeatMs: 60_000,
maxIdlePolls: 0,
} as const;
export type WorkerConfig = {
store: ConvexStore;
cipher: TokenCipher;
sso: EveSso;
esi: EsiClient;
workerId: string;
pollMs: number;
leaseMs: number;
heartbeatMs: number;
sleep: (ms: number) => Promise<void>;
};
function fixedFailure(): string { return 'Asset synchronization failed'; }
const DEFAULT_SYNC_COOLDOWN_GROUP = 'esi:assets';
function cooldownContext(userId: string, characterId: number): string { return `character:${userId}:${characterId}`; }
export class MutationUncertainError extends Error {
constructor(cause: unknown) { super('Synchronization mutation acknowledgement was uncertain'); this.name = 'MutationUncertainError'; this.cause = cause; }
}
function permanentMutationFailure(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return /lease is not valid|authorization (?:is )?stale|authorization changed|generation is not current|already leased|Invalid asset chunk|Asset character mismatch|chunks must be staged|chunk already exists|Snapshot is not staging|Snapshot is incomplete|Invalid snapshot|not active|missing|Invalid refreshed scopes/.test(message);
}
async function retryMutation<T>(operation: () => Promise<T>, renew?: () => Promise<boolean>): Promise<T> {
let lastError: unknown;
for (let attempt = 0; attempt < 2; attempt += 1) {
try { return await operation(); }
catch (error) {
lastError = error;
if (permanentMutationFailure(error)) throw error;
if (attempt === 0 && renew) {
try { if (!(await renew())) throw new Error('Synchronization lease is not valid'); }
catch (renewError) {
if (permanentMutationFailure(renewError)) throw renewError;
lastError = renewError;
}
}
}
}
throw new MutationUncertainError(lastError);
}
/** Process one at-least-once delivery. Every write is fenced by the durable lease. */
export async function processSyncJob(config: WorkerConfig): Promise<boolean> {
const claimed = await config.store.claimSyncJob(config.workerId, config.leaseMs);
if (!claimed) return false;
let lostLease = false;
let refreshLease: RefreshLease | null = null;
let refreshAttempted = false;
const heartbeat = setInterval(() => {
void config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs).then((ok) => {
if (!ok) lostLease = true;
}).catch(() => { lostLease = true; });
}, config.heartbeatMs);
const stopHeartbeat = () => clearInterval(heartbeat);
try {
const refresh = await config.store.acquireRefreshLease(claimed.userId, claimed.characterId, config.workerId);
refreshLease = refresh;
if (!refresh) {
await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, Date.now() + 1_000);
return true;
}
let previous: VerifiedGrant;
try {
previous = await config.cipher.open<VerifiedGrant>(refresh.encryptedTokens, `character:${claimed.characterId}`);
if (previous.characterId !== claimed.characterId) throw new Error('authorization mismatch');
} catch {
await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {});
return true;
}
const needsRefresh = refresh.tokenExpiresAt <= Date.now() + 60_000;
let grant: VerifiedGrant = previous;
let authorityVersion = { grantRevision: refresh.grantRevision, authorizationEpoch: refresh.authorizationEpoch, credentialVersion: refresh.credentialVersion, scopes: refresh.scopes };
if (!needsRefresh) {
// A lease acquired for a still-valid token is only a serialization guard;
// release it before paging so another job is not blocked for two minutes.
await retryMutation(() => config.store.releaseRefreshLease
? config.store.releaseRefreshLease(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId)
: Promise.resolve(true));
refreshLease = null;
}
if (needsRefresh) {
try {
const dispatched = await config.store.markRefreshDispatched(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId);
if (!dispatched) {
await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {});
return true;
}
refreshAttempted = true;
// Exactly one remote refresh attempt. An error is treated as uncertain:
// rotating refresh tokens cannot safely be replayed after a lost result.
grant = await config.sso.refresh(previous.refreshToken);
} catch {
await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {});
return true;
}
if (grant.characterId !== claimed.characterId || grant.ownerHash !== previous.ownerHash) {
await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {});
return true;
}
const committed = await config.store.commitRefresh(
claimed.userId,
claimed.characterId,
refresh.leaseToken,
config.workerId,
refresh.expectedVersion,
{ encryptedTokens: await config.cipher.seal(grant, `character:${claimed.characterId}`), tokenExpiresAt: grant.expiresAt, scopes: grant.scopes, jobId: claimed.jobId, jobLeaseToken: claimed.leaseToken },
);
if (!committed) {
await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refresh.leaseToken, config.workerId, refresh.expectedVersion).catch(() => {});
await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {});
return true;
}
authorityVersion = committed;
// A refreshed credential cannot publish into a generation fenced to the
// old credential version. Discard only that staging generation.
if (claimed.generation !== undefined) await retryMutation(
() => config.store.restartSyncJobGeneration(claimed.jobId, claimed.leaseToken),
() => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs),
);
}
const currentAuthority = await retryMutation(
() => config.store.refreshSyncJobAuthority(claimed.jobId, claimed.leaseToken, authorityVersion),
() => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs),
);
if (!grant.scopes.includes('esi-assets.read_assets.v1') || !currentAuthority.scopes.includes('esi-assets.read_assets.v1') || lostLease) {
await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {});
return true;
}
authorityVersion = currentAuthority;
const generation = await retryMutation(
() => config.store.beginSyncJob(claimed.jobId, claimed.leaseToken),
() => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs),
);
const context = cooldownContext(claimed.userId, claimed.characterId);
const cooldown = await config.store.cooldown(context, claimed.cooldownGroup);
if (cooldown && cooldown.until > Date.now()) await config.sleep(cooldown.until - Date.now());
const authority = {
userId: claimed.userId,
principal: grant.ownerHash,
characterId: claimed.characterId,
scopes: authorityVersion.scopes,
grantRevision: authorityVersion.grantRevision,
authorizationEpoch: authorityVersion.authorizationEpoch,
credentialVersion: authorityVersion.credentialVersion,
} as const;
const snapshot = await config.esi.getAssets(claimed.characterId, grant.accessToken, authority);
const chunks = Math.ceil(snapshot.assets.length / 250);
for (let index = 0; index < chunks; index += 1) {
if (lostLease) throw new Error('lease lost');
await retryMutation(
() => config.store.stageSyncAssets(claimed.jobId, claimed.leaseToken, index, snapshot.assets.slice(index * 250, (index + 1) * 250)),
() => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs),
);
}
await retryMutation(
() => config.store.commitSyncJob(claimed.jobId, claimed.leaseToken, chunks, snapshot.syncedAt, snapshot.expiresAt),
() => config.store.renewSyncJob(claimed.jobId, claimed.leaseToken, config.workerId, config.leaseMs),
);
return true;
} catch (error) {
if (refreshAttempted && refreshLease) await config.store.markRefreshUncertain(claimed.userId, claimed.characterId, refreshLease.leaseToken, config.workerId, refreshLease.expectedVersion).catch(() => {});
if (error instanceof MutationUncertainError) {
await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, Date.now() + 1_000).catch(() => {});
} else if (error instanceof EsiRateLimitError) {
const retryAt = Math.min(Date.now() + 24 * 60 * 60_000, Math.max(Date.now() + 1_000, error.retryAt));
await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, retryAt, { context: cooldownContext(claimed.userId, claimed.characterId), until: retryAt, group: error.cooldown.group ?? DEFAULT_SYNC_COOLDOWN_GROUP }).catch(() => {});
} else if (error instanceof EsiError && (error.code === 'network' || error.code === 'deadline' || (error instanceof EsiHttpError && (error.status === 408 || error.status === 420 || error.status === 425 || error.status === 429 || error.status >= 500)))) {
const retryAt = Math.min(Date.now() + 24 * 60 * 60_000, Date.now() + 1_000 * 2 ** Math.max(0, claimed.attempts - 1));
await config.store.deferSyncJob(claimed.jobId, claimed.leaseToken, retryAt).catch(() => {});
} else {
await config.store.failSyncJob(claimed.jobId, claimed.leaseToken).catch(() => {});
}
return true;
} finally {
stopHeartbeat();
}
}
function env(name: string): string {
const value = process.env[name];
if (!value) throw new Error(`Sync worker configuration missing: ${name}`);
return value;
}
export async function runSyncWorker(): Promise<void> {
const workerId = process.env.SYNC_WORKER_ID ?? `sync-worker-${crypto.randomUUID()}`;
const pollMs = Math.max(250, Math.min(30_000, Number(process.env.SYNC_WORKER_POLL_MS ?? SYNC_WORKER_DEFAULTS.pollMs)));
const leaseMs = Math.max(60_000, Math.min(60 * 60_000, Number(process.env.SYNC_WORKER_LEASE_MS ?? SYNC_WORKER_DEFAULTS.leaseMs)));
const heartbeatMs = Math.max(10_000, Math.min(10 * 60_000, Number(process.env.SYNC_WORKER_HEARTBEAT_MS ?? SYNC_WORKER_DEFAULTS.heartbeatMs)));
const store = new ConvexStore(env('CONVEX_URL'), undefined, env('CONVEX_SYNC_WORKER_SECRET'));
const cipher = new TokenCipher(env('TOKEN_ENCRYPTION_KEY'));
const userAgent = env('ESI_USER_AGENT');
const sso = new EveSso({ clientId: env('EVE_CLIENT_ID'), clientSecret: process.env.EVE_CLIENT_SECRET || undefined, redirectUri: process.env.EVE_CALLBACK_URL ?? 'http://localhost/unused-worker-callback', userAgent });
const esi = new EsiClient({ userAgent });
const config: WorkerConfig = { store, cipher, sso, esi, workerId, pollMs, leaseMs, heartbeatMs, sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) };
if (process.env.SYNC_WORKER_STARTUP_CHECK === '1') return;
for (;;) {
const handled = await processSyncJob(config);
if (!handled) await config.sleep(pollMs);
}
}
if (import.meta.main) await runSyncWorker();