feat: add durable asset synchronization worker
This commit is contained in:
+608
-4
@@ -20,6 +20,8 @@ const MAX_CHUNKS = 4_096;
|
||||
const MAX_ERROR_LENGTH = 160;
|
||||
const OAUTH_MAX_LIFETIME_MS = 10 * 60 * 1000;
|
||||
const SNAPSHOT_MAX_LIFETIME_MS = 60 * 60 * 1000;
|
||||
const MAX_USER_ACTIVE_SYNC_JOBS = 8;
|
||||
const SYNC_PROVIDER_GROUP = 'esi:assets';
|
||||
const DATASET_SCOPE_REQUIREMENTS: Record<string, readonly string[]> = {
|
||||
assets: ['esi-assets.read_assets.v1'],
|
||||
wallet_balance: ['esi-wallet.read_character_wallet.v1'],
|
||||
@@ -33,6 +35,7 @@ const CONSENT_INTENTS = ['initial_login', 'link_character', 'upgrade_dataset', '
|
||||
const CONSENT_CAPABILITIES = Object.keys(CAPABILITY_DEFINITIONS) as Capability[];
|
||||
|
||||
const bridgeArgs = { bridgeSecret: v.string() };
|
||||
const workerBridgeArgs = { workerSecret: v.string() };
|
||||
const characterArgs = { userId: v.string(), characterId: v.number() };
|
||||
|
||||
const assetPageResultValidator = v.object({
|
||||
@@ -57,6 +60,12 @@ function assertBridge(secret: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function assertSyncWorker(secret: string): void {
|
||||
const expected = process.env.CONVEX_SYNC_WORKER_SECRET;
|
||||
if (!expected || expected === process.env.CONVEX_BRIDGE_SECRET || !constantTimeEqual(secret, expected)) throw new Error("Sync worker authentication failed");
|
||||
assertDeploymentCapability('asset_sync');
|
||||
}
|
||||
|
||||
function assertDeploymentCapability(capability: 'asset_sync'): void {
|
||||
let enabled = false;
|
||||
try {
|
||||
@@ -81,6 +90,19 @@ function safeSyncError(_error: string): string {
|
||||
return "Asset synchronization failed";
|
||||
}
|
||||
|
||||
function workerJob(job: any): boolean {
|
||||
return job?.kind === 'asset_sync' && job?.dataset === 'assets' && job?.authority === 'user' && job?.ownerKind === 'user';
|
||||
}
|
||||
|
||||
function authorityMatches(character: any, job: any): boolean {
|
||||
return Boolean(character && character.authorizationStatus === 'active'
|
||||
&& character.scopes.includes('esi-assets.read_assets.v1')
|
||||
&& character.grantRevision === job.grantRevision
|
||||
&& character.authorizationEpoch === job.authorizationEpoch
|
||||
&& character.credentialVersion === job.credentialVersion
|
||||
&& scopesEqual(character.scopes, job.scopes ?? []));
|
||||
}
|
||||
|
||||
function newIdentifier(prefix: string): string {
|
||||
return `${prefix}_${crypto.randomUUID()}`;
|
||||
}
|
||||
@@ -109,6 +131,11 @@ function validActualScopes(scopes: readonly string[]): boolean {
|
||||
return new Set(scopes).size === scopes.length && scopes.every((scope) => SERVER_SCOPE_ALLOWLIST.includes(scope));
|
||||
}
|
||||
|
||||
function scopesSubset(subset: readonly string[], superset: readonly string[]): boolean {
|
||||
const allowed = new Set(superset);
|
||||
return subset.every((scope) => allowed.has(scope));
|
||||
}
|
||||
|
||||
function datasetStatusesFor(character: any): any[] {
|
||||
return character.datasetStatuses;
|
||||
}
|
||||
@@ -138,7 +165,7 @@ function updateDatasetStatuses(
|
||||
invalidatedAt: timestamp,
|
||||
...(dataset === 'assets' ? {} : {}),
|
||||
});
|
||||
if (dataset === 'assets' && (old?.status === 'ready' || old?.generation !== undefined || character.activeGeneration !== undefined)) assetsInvalidated = true;
|
||||
if (dataset === 'assets') assetsInvalidated = true;
|
||||
continue;
|
||||
}
|
||||
// A newly authorized dataset has no coverage. Existing coverage remains
|
||||
@@ -458,7 +485,7 @@ export const bridgeCompleteLogin = mutationGeneric({
|
||||
credentialVersion,
|
||||
authorizationEpoch,
|
||||
datasetStatuses: updated.statuses,
|
||||
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}),
|
||||
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}),
|
||||
};
|
||||
if (character) {
|
||||
await ctx.db.patch(character._id, basePatch);
|
||||
@@ -492,6 +519,7 @@ export const bridgeCompleteLogin = mutationGeneric({
|
||||
version: credentialVersion,
|
||||
});
|
||||
}
|
||||
if (character && (grantChanged || updated.assetsInvalidated || credentialVersion !== character.credentialVersion)) await invalidateCharacterJobs(ctx, resolvedUserId, args.characterId, timestamp);
|
||||
|
||||
const sessionId = newIdentifier("session");
|
||||
await ctx.db.insert("sessions", {
|
||||
@@ -546,7 +574,7 @@ export const bridgeSaveTokens = mutationGeneric({
|
||||
assertBridge(args.bridgeSecret);
|
||||
if (!validActualScopes(args.scopes)) return false;
|
||||
const character = await findCharacter(ctx, args.userId, args.characterId);
|
||||
if (!character || character.authorizationStatus !== "active") return false;
|
||||
if (!character || character.authorizationStatus !== "active" || !scopesSubset(args.scopes, character.scopes ?? [])) return false;
|
||||
const record = await ctx.db
|
||||
.query("tokenRecords")
|
||||
.withIndex("by_user_character", (q: any) =>
|
||||
@@ -568,8 +596,9 @@ export const bridgeSaveTokens = mutationGeneric({
|
||||
grantRevision: nextRevision,
|
||||
credentialVersion: nextVersion,
|
||||
datasetStatuses: updated.statuses,
|
||||
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined } : {}),
|
||||
...(updated.assetsInvalidated ? { state: "error" as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}),
|
||||
});
|
||||
if (updated.assetsInvalidated || nextRevision !== character.grantRevision || nextVersion !== character.credentialVersion) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now());
|
||||
if (args.returnAuthority) {
|
||||
return {
|
||||
saved: true,
|
||||
@@ -973,6 +1002,581 @@ export const bridgeFailSync = mutationGeneric({
|
||||
},
|
||||
});
|
||||
|
||||
/** Return only fields safe for an authenticated status response. */
|
||||
function safeSyncJob(job: any): any {
|
||||
const knownErrors = new Set(['Synchronization cancelled', 'Synchronization scheduled for retry', 'Authorization is no longer active', 'Asset synchronization failed']);
|
||||
const error = typeof job.lastError === 'string' && knownErrors.has(job.lastError) ? job.lastError : undefined;
|
||||
return {
|
||||
id: job._id,
|
||||
dataset: job.dataset ?? 'assets',
|
||||
characterId: job.characterId,
|
||||
status: job.status,
|
||||
attempts: job.attempts ?? 0,
|
||||
dueAt: job.dueAt ?? job.createdAt,
|
||||
createdAt: job.createdAt,
|
||||
updatedAt: job.updatedAt,
|
||||
leaseExpiresAt: job.status === 'running' ? (job.leaseExpiresAt ?? null) : null,
|
||||
cancelRequested: job.cancelRequested === true,
|
||||
...(error === undefined ? {} : { error }),
|
||||
};
|
||||
}
|
||||
|
||||
function safeJobError(): string { return 'Asset synchronization failed'; }
|
||||
function activeJobStatus(status: string): boolean { return status === 'queued' || status === 'running'; }
|
||||
|
||||
async function findJob(ctx: any, jobId: string): Promise<any | null> {
|
||||
return await ctx.db.get(jobId as any);
|
||||
}
|
||||
|
||||
async function markTerminalSyncFailure(ctx: any, job: any, error = safeSyncError('terminal failure')): Promise<void> {
|
||||
if (job.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error });
|
||||
}
|
||||
const character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (!character) return;
|
||||
const assetWasSyncing = character.syncGeneration === job.generation || character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && (entry.status === 'syncing' || entry.status === 'queued'));
|
||||
if (!assetWasSyncing) return;
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
|
||||
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error }
|
||||
: entry);
|
||||
// Never clear activeGeneration/syncedAt/expiresAt: the last complete snapshot
|
||||
// remains the only readable publication until a replacement succeeds.
|
||||
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error });
|
||||
}
|
||||
|
||||
async function clearMatchingSyncPointer(ctx: any, character: any, generation: string, error = 'Authorization is no longer active'): Promise<boolean> {
|
||||
if (!character || character.syncGeneration !== generation) return false;
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
|
||||
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error, generation: character.activeGeneration ?? undefined }
|
||||
: entry);
|
||||
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function invalidateCharacterJobs(ctx: any, userId: string, characterId: number, timestamp: number): Promise<void> {
|
||||
const character = await findCharacter(ctx, userId, characterId);
|
||||
const jobs = await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', userId)).collect();
|
||||
let staleSyncState = false;
|
||||
for (const job of jobs) {
|
||||
if (workerJob(job) && job.characterId === characterId && activeJobStatus(job.status)) {
|
||||
if (job.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', userId).eq('characterId', characterId).eq('generation', job.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
|
||||
}
|
||||
if (character && job.generation !== undefined && character.syncGeneration === job.generation) staleSyncState = true;
|
||||
await ctx.db.patch(job._id, {
|
||||
status: 'quarantined', updatedAt: timestamp, leaseExpiresAt: undefined,
|
||||
leaseToken: undefined, workerId: undefined, cancelRequested: undefined,
|
||||
lastError: 'Authorization is no longer active', finishedAt: timestamp,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (character && staleSyncState && character.syncGeneration !== undefined) await clearMatchingSyncPointer(ctx, character, character.syncGeneration);
|
||||
}
|
||||
|
||||
/** Server-derived enqueue. The caller supplies a session proof, never a job authority. */
|
||||
export const bridgeEnqueueAssetSync = mutationGeneric({
|
||||
args: {
|
||||
...bridgeArgs,
|
||||
userId: v.string(),
|
||||
characterId: v.number(),
|
||||
initiatingSessionHash: v.string(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
assertBridge(args.bridgeSecret);
|
||||
assertDeploymentCapability('asset_sync');
|
||||
const timestamp = now();
|
||||
const initiating = await findSessionByHash(ctx, args.initiatingSessionHash);
|
||||
if (!isUsableSession(initiating, timestamp) || initiating.userId !== args.userId) throw new Error('Session is no longer valid');
|
||||
const character = await findCharacter(ctx, args.userId, args.characterId);
|
||||
if (!character || character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) {
|
||||
throw new Error('Asset authorization is not active');
|
||||
}
|
||||
const dedupeKey = `${args.userId}:character:${args.characterId}:assets`;
|
||||
const existing = await ctx.db.query('jobs').withIndex('by_dedupe', (q: any) => q.eq('dedupeKey', dedupeKey)).collect();
|
||||
const active = existing.find((job: any) => activeJobStatus(job.status));
|
||||
if (active) return safeSyncJob(active);
|
||||
const queuedJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job));
|
||||
const runningJobs = (await ctx.db.query('jobs').withIndex('by_user_asset_status', (q: any) => q.eq('userId', args.userId).eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'running')).take(MAX_USER_ACTIVE_SYNC_JOBS + 1)).filter((job: any) => workerJob(job));
|
||||
if (queuedJobs.length + runningJobs.length >= MAX_USER_ACTIVE_SYNC_JOBS) throw new Error('Synchronization queue is full');
|
||||
const maxAttempts = 5;
|
||||
const job = await ctx.db.insert('jobs', {
|
||||
userId: args.userId,
|
||||
kind: 'asset_sync',
|
||||
dataset: 'assets',
|
||||
status: 'queued',
|
||||
characterId: args.characterId,
|
||||
dedupeKey,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
dueAt: timestamp,
|
||||
attempts: 0,
|
||||
maxAttempts,
|
||||
priority: 0,
|
||||
ownerKind: 'user',
|
||||
authority: 'user',
|
||||
grantRevision: character.grantRevision,
|
||||
authorizationEpoch: character.authorizationEpoch,
|
||||
credentialVersion: character.credentialVersion,
|
||||
scopes: [...character.scopes],
|
||||
cooldownGroup: SYNC_PROVIDER_GROUP,
|
||||
cancelRequested: false,
|
||||
});
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
|
||||
? { ...entry, status: 'queued', grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion, error: undefined }
|
||||
: entry);
|
||||
await ctx.db.patch(character._id, { state: 'queued', datasetStatuses: statuses, error: undefined });
|
||||
return safeSyncJob({ _id: job, dataset: 'assets', characterId: args.characterId, status: 'queued', attempts: 0, dueAt: timestamp, createdAt: timestamp, updatedAt: timestamp, cancelRequested: false });
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeSyncJob = queryGeneric({
|
||||
args: { ...bridgeArgs, userId: v.string(), jobId: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertBridge(args.bridgeSecret);
|
||||
const job = await findJob(ctx, args.jobId);
|
||||
if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null;
|
||||
return safeSyncJob(job);
|
||||
},
|
||||
});
|
||||
|
||||
/** Cancellation is authenticated against the initiating session and fences a running lease immediately. */
|
||||
export const bridgeCancelSyncJob = mutationGeneric({
|
||||
args: { ...bridgeArgs, userId: v.string(), jobId: v.string(), initiatingSessionHash: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertBridge(args.bridgeSecret);
|
||||
assertDeploymentCapability('asset_sync');
|
||||
const initiating = await findSessionByHash(ctx, args.initiatingSessionHash);
|
||||
if (!isUsableSession(initiating) || initiating.userId !== args.userId) throw new Error('Session is no longer valid');
|
||||
const job = await findJob(ctx, args.jobId);
|
||||
if (!job || job.userId !== args.userId || job.kind !== 'asset_sync') return null;
|
||||
if (!activeJobStatus(job.status)) return safeSyncJob(job);
|
||||
const timestamp = now();
|
||||
const character = job.characterId === undefined ? null : await findCharacter(ctx, args.userId, job.characterId);
|
||||
if (job.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', args.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Synchronization cancelled' });
|
||||
}
|
||||
await ctx.db.patch(job._id, {
|
||||
status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp,
|
||||
leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined,
|
||||
cancelRequested: true, lastError: 'Synchronization cancelled',
|
||||
});
|
||||
if (character && character.syncGeneration === job.generation) {
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets'
|
||||
? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' }
|
||||
: entry);
|
||||
await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Synchronization cancelled' });
|
||||
} else if (character && character.datasetStatuses?.some((entry: any) => entry.dataset === 'assets' && entry.status === 'queued')) {
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: 'Synchronization cancelled' } : entry);
|
||||
await ctx.db.patch(character._id, { state: character.activeGeneration ? 'error' : 'error', datasetStatuses: statuses, error: 'Synchronization cancelled' });
|
||||
}
|
||||
return safeSyncJob({ ...job, status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, cancelRequested: true, lastError: 'Synchronization cancelled' });
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Claim and recover in one transaction. Expired leases are requeued before a
|
||||
* candidate is selected, making process termination recoverable without a
|
||||
* second scheduler or an in-memory queue.
|
||||
*/
|
||||
export const bridgeClaimSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, workerId: v.string(), leaseMs: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
if (!args.workerId || args.workerId.length > 160) throw new Error('Invalid worker identity');
|
||||
const timestamp = now();
|
||||
const leaseMs = args.leaseMs ?? 10 * 60 * 1000;
|
||||
if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) throw new Error('Invalid sync lease duration');
|
||||
const expired = await ctx.db.query('jobs').withIndex('by_lease_expires_at', (q: any) => q.lte('leaseExpiresAt', timestamp)).take(100);
|
||||
for (const job of expired) {
|
||||
if (!workerJob(job) || job.status !== 'running' || (job.leaseExpiresAt ?? Infinity) > timestamp) continue;
|
||||
if (job.cancelRequested === true) {
|
||||
await ctx.db.patch(job._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization cancelled' });
|
||||
} else if ((job.attempts ?? 0) >= (job.maxAttempts ?? 5)) {
|
||||
const character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (!authorityMatches(character, job)) {
|
||||
if (job.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
|
||||
await clearMatchingSyncPointer(ctx, character, job.generation);
|
||||
}
|
||||
await ctx.db.patch(job._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Authorization is no longer active' });
|
||||
continue;
|
||||
}
|
||||
await markTerminalSyncFailure(ctx, job);
|
||||
await ctx.db.patch(job._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
|
||||
} else {
|
||||
await ctx.db.patch(job._id, { status: 'queued', updatedAt: timestamp, dueAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined });
|
||||
}
|
||||
}
|
||||
const candidates = await ctx.db.query('jobs').withIndex('by_asset_sync_queue', (q: any) => q.eq('kind', 'asset_sync').eq('dataset', 'assets').eq('authority', 'user').eq('ownerKind', 'user').eq('status', 'queued').lte('dueAt', timestamp)).order('asc').take(100);
|
||||
if (candidates.length === 0) return null;
|
||||
// Bounded fair selection: an account with an in-flight job loses to an
|
||||
// account with no in-flight work; due/creation order breaks ties.
|
||||
const supportedCandidates = candidates.filter((candidate: any) => workerJob(candidate));
|
||||
if (supportedCandidates.length === 0) return null;
|
||||
const runningByUser = new Map<string, number>();
|
||||
for (const candidate of supportedCandidates) {
|
||||
if (runningByUser.has(candidate.userId)) continue;
|
||||
const running = (await ctx.db.query('jobs').withIndex('by_user_status', (q: any) => q.eq('userId', candidate.userId).eq('status', 'running')).take(20)).filter((job: any) => workerJob(job));
|
||||
runningByUser.set(candidate.userId, running.length);
|
||||
}
|
||||
const ordered = [...supportedCandidates].sort((left: any, right: any) =>
|
||||
(runningByUser.get(left.userId)! - runningByUser.get(right.userId)!)
|
||||
|| ((left.priority ?? 0) - (right.priority ?? 0))
|
||||
|| ((left.dueAt ?? left.createdAt) - (right.dueAt ?? right.createdAt))
|
||||
|| ((left.lastClaimedAt ?? 0) - (right.lastClaimedAt ?? 0))
|
||||
|| String(left._id).localeCompare(String(right._id)));
|
||||
for (const candidate of ordered) {
|
||||
if (candidate.cancelRequested === true) {
|
||||
await ctx.db.patch(candidate._id, { status: 'cancelled', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Synchronization cancelled' });
|
||||
continue;
|
||||
}
|
||||
if (candidate.cooldownGroup) {
|
||||
const shared = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${candidate.cooldownGroup}`)).first();
|
||||
if (shared && shared.until > timestamp) continue;
|
||||
}
|
||||
const character = candidate.characterId === undefined ? null : await findCharacter(ctx, candidate.userId, candidate.characterId);
|
||||
const authorityCurrent = authorityMatches(character, candidate);
|
||||
if (!authorityCurrent) {
|
||||
if (candidate.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', candidate.userId).eq('characterId', candidate.characterId).eq('generation', candidate.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization is no longer active' });
|
||||
await clearMatchingSyncPointer(ctx, character, candidate.generation);
|
||||
}
|
||||
await ctx.db.patch(candidate._id, { status: 'quarantined', updatedAt: timestamp, finishedAt: timestamp, lastError: 'Authorization is no longer active' });
|
||||
continue;
|
||||
}
|
||||
const leaseToken = newIdentifier('lease');
|
||||
const attempts = (candidate.attempts ?? 0) + 1;
|
||||
await ctx.db.patch(candidate._id, { status: 'running', attempts, updatedAt: timestamp, startedAt: candidate.startedAt ?? timestamp, dueAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken, workerId: args.workerId, lastClaimedAt: timestamp });
|
||||
return { ...safeSyncJob({ ...candidate, status: 'running', attempts, updatedAt: timestamp, leaseExpiresAt: timestamp + leaseMs, leaseToken }), jobId: candidate._id, leaseToken, userId: candidate.userId, characterId: candidate.characterId, grantRevision: candidate.grantRevision, authorizationEpoch: candidate.authorizationEpoch, credentialVersion: candidate.credentialVersion, scopes: candidate.scopes ?? [], ...(candidate.generation === undefined ? {} : { generation: candidate.generation }), cooldownGroup: candidate.cooldownGroup ?? SYNC_PROVIDER_GROUP };
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeRenewSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), workerId: v.string(), leaseMs: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const job = await findJob(ctx, args.jobId);
|
||||
if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== args.leaseToken || job.workerId !== args.workerId) return false;
|
||||
if (job.cancelRequested === true || (job.leaseExpiresAt ?? 0) <= now()) return false;
|
||||
const leaseMs = args.leaseMs ?? 10 * 60 * 1000;
|
||||
if (!Number.isSafeInteger(leaseMs) || leaseMs < 60_000 || leaseMs > 60 * 60 * 1000) return false;
|
||||
await ctx.db.patch(job._id, { leaseExpiresAt: now() + leaseMs, updatedAt: now() });
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
async function jobLease(ctx: any, jobId: string, leaseToken: string): Promise<any> {
|
||||
const job = await findJob(ctx, jobId);
|
||||
if (!job || !workerJob(job) || job.status !== 'running' || job.leaseToken !== leaseToken || (job.leaseExpiresAt ?? 0) <= now()) throw new Error('Synchronization lease is not valid');
|
||||
return job;
|
||||
}
|
||||
|
||||
async function jobCharacter(ctx: any, job: any): Promise<any> {
|
||||
const character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (!authorityMatches(character, job)) {
|
||||
throw new Error('Synchronization authorization is stale');
|
||||
}
|
||||
return character;
|
||||
}
|
||||
|
||||
export const bridgeRefreshSyncJobAuthority = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), expectedGrantRevision: v.number(), expectedAuthorizationEpoch: v.number(), expectedCredentialVersion: v.number(), expectedScopes: v.array(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const job = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
const character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
const expected = { grantRevision: args.expectedGrantRevision, authorizationEpoch: args.expectedAuthorizationEpoch, credentialVersion: args.expectedCredentialVersion, scopes: args.expectedScopes };
|
||||
if (!authorityMatches(character, expected)) throw new Error('Synchronization authorization changed');
|
||||
if (!authorityMatches(character, job)) throw new Error('Synchronization job authority is stale');
|
||||
return expected;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeRestartSyncJobGeneration = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const job = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
if (job.generation === undefined) return null;
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: 'Authorization refreshed before publication' });
|
||||
const character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (character?.syncGeneration === job.generation) await ctx.db.patch(character._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined, state: 'queued' });
|
||||
await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() });
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeBeginSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const job = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
let character = await jobCharacter(ctx, job);
|
||||
if (job.generation !== undefined) {
|
||||
const existing = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (existing && existing.status === 'staging' && existing.grantRevision === job.grantRevision && existing.authorizationEpoch === job.authorizationEpoch && existing.credentialVersion === job.credentialVersion) return job.generation;
|
||||
if (existing?.status === 'staging') await ctx.db.patch(existing._id, { status: 'failed', error: 'Synchronization authority changed' });
|
||||
await ctx.db.patch(job._id, { generation: undefined, updatedAt: now() });
|
||||
const oldCharacter = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (oldCharacter?.syncGeneration === job.generation) await ctx.db.patch(oldCharacter._id, { syncGeneration: undefined, syncLeaseExpiresAt: undefined });
|
||||
// The character read above predates the atomic cleanup. Re-read it before
|
||||
// checking the lease, otherwise a reclaimed job sees its own old pointer
|
||||
// and incorrectly reports an active synchronization.
|
||||
character = await findCharacter(ctx, job.userId, job.characterId);
|
||||
if (!character || !authorityMatches(character, job)) throw new Error('Synchronization authorization is stale');
|
||||
}
|
||||
if (character.syncGeneration && (character.syncLeaseExpiresAt ?? 0) > now()) throw new Error('Character synchronization is already leased');
|
||||
const generation = newIdentifier('generation');
|
||||
await ctx.db.insert('assetGenerations', { userId: job.userId, characterId: job.characterId, generation, status: 'staging', stagedChunkCount: 0, grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion });
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'syncing', grantRevision: job.grantRevision, authorizationEpoch: job.authorizationEpoch, credentialVersion: job.credentialVersion, error: undefined } : entry);
|
||||
await ctx.db.patch(character._id, { state: 'syncing', datasetStatuses: statuses, syncGeneration: generation, syncLeaseExpiresAt: now() + 10 * 60 * 1000, error: undefined });
|
||||
await ctx.db.patch(job._id, { generation, updatedAt: now() });
|
||||
return generation;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeStageSyncAssets = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), index: v.number(), assets: v.array(assetValidator) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
if (!Number.isInteger(args.index) || args.index < 0 || args.assets.length > MAX_ASSET_CHUNK) throw new Error('Invalid asset chunk');
|
||||
const job = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
const character = await jobCharacter(ctx, job);
|
||||
if (args.assets.some((asset: any) => asset.characterId !== job.characterId)) throw new Error('Asset character mismatch');
|
||||
if (!job.generation || character.syncGeneration !== job.generation) throw new Error('Synchronization generation is not current');
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation)).first();
|
||||
if (!generation || generation.status !== 'staging') throw new Error('Snapshot is not staging');
|
||||
const stagedChunkCount = generation.stagedChunkCount ?? 0;
|
||||
if (stagedChunkCount >= MAX_CHUNKS || args.index > stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order');
|
||||
const existing = await ctx.db.query('assetChunks').withIndex('by_snapshot_index', (q: any) => q.eq('userId', job.userId).eq('characterId', job.characterId).eq('generation', job.generation).eq('index', args.index)).first();
|
||||
if (existing) {
|
||||
// Convex may commit a mutation while its network response is lost. A
|
||||
// duplicate delivery is accepted only when its payload is identical.
|
||||
if (JSON.stringify(existing.assets) !== JSON.stringify(args.assets)) throw new Error('Asset chunk already exists');
|
||||
return null;
|
||||
}
|
||||
if (args.index !== stagedChunkCount) throw new Error('Asset chunks must be staged exactly once and in order');
|
||||
await ctx.db.insert('assetChunks', { userId: job.userId, characterId: job.characterId, generation: job.generation, index: args.index, assets: args.assets });
|
||||
await ctx.db.patch(generation._id, { stagedChunkCount: stagedChunkCount + 1 });
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
function validJobSnapshotTimes(syncedAt: number, expiresAt: number, timestamp: number): boolean {
|
||||
return Number.isFinite(syncedAt) && syncedAt >= 0 && syncedAt <= timestamp && Number.isFinite(expiresAt) && expiresAt > timestamp && expiresAt > syncedAt && expiresAt - syncedAt <= SNAPSHOT_MAX_LIFETIME_MS;
|
||||
}
|
||||
|
||||
export const bridgeCommitSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), chunkCount: v.number(), syncedAt: v.number(), expiresAt: v.number() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const job = await findJob(ctx, args.jobId);
|
||||
if (job?.status === 'succeeded' && job.generation !== undefined) return true;
|
||||
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
const character = await jobCharacter(ctx, leased);
|
||||
if (!leased.generation) throw new Error('Synchronization generation is not current');
|
||||
if (!Number.isInteger(args.chunkCount) || args.chunkCount < 0 || args.chunkCount > MAX_CHUNKS || !validJobSnapshotTimes(args.syncedAt, args.expiresAt, now())) throw new Error('Invalid snapshot');
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first();
|
||||
if (!generation || generation.status !== 'staging' || (generation.stagedChunkCount ?? 0) !== args.chunkCount) throw new Error('Snapshot is incomplete');
|
||||
if (generation.grantRevision !== character.grantRevision || generation.authorizationEpoch !== character.authorizationEpoch || generation.credentialVersion !== character.credentialVersion) throw new Error('Synchronization authorization is stale');
|
||||
await ctx.db.patch(generation._id, { status: 'complete', chunkCount: args.chunkCount, syncedAt: args.syncedAt, expiresAt: args.expiresAt });
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: 'ready', generation: leased.generation, syncedAt: args.syncedAt, error: undefined } : entry);
|
||||
await ctx.db.patch(character._id, { state: 'ready', datasetStatuses: statuses, syncedAt: args.syncedAt, expiresAt: args.expiresAt, activeGeneration: leased.generation, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: undefined });
|
||||
await ctx.db.patch(leased._id, { status: 'succeeded', updatedAt: now(), finishedAt: now(), leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, cancelRequested: undefined, lastError: undefined });
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeFailSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
const character = await jobCharacter(ctx, leased);
|
||||
const timestamp = now();
|
||||
if (leased.generation !== undefined) {
|
||||
const generation = await ctx.db.query('assetGenerations').withIndex('by_user_character_generation', (q: any) => q.eq('userId', leased.userId).eq('characterId', leased.characterId).eq('generation', leased.generation)).first();
|
||||
if (generation?.status === 'staging') await ctx.db.patch(generation._id, { status: 'failed', error: safeJobError() });
|
||||
}
|
||||
const statuses = datasetStatusesFor(character).map((entry: any) => entry.dataset === 'assets' ? { ...entry, status: character.activeGeneration ? 'error' : 'blocked', error: safeJobError() } : entry);
|
||||
await ctx.db.patch(character._id, { state: 'error', datasetStatuses: statuses, error: safeJobError(), syncGeneration: undefined, syncLeaseExpiresAt: undefined });
|
||||
await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
/** Reschedule transient provider/cooldown failures without exposing their raw text. */
|
||||
export const bridgeDeferSyncJob = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, jobId: v.string(), leaseToken: v.string(), dueAt: v.number(), cooldownContext: v.optional(v.string()), cooldownUntil: v.optional(v.number()), cooldownGroup: v.optional(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const leased = await jobLease(ctx, args.jobId, args.leaseToken);
|
||||
const timestamp = now();
|
||||
if (!Number.isFinite(args.dueAt) || args.dueAt < timestamp || args.dueAt > timestamp + 24 * 60 * 60 * 1000) throw new Error('Invalid retry due time');
|
||||
if ((leased.attempts ?? 0) >= (leased.maxAttempts ?? 5)) {
|
||||
await jobCharacter(ctx, leased);
|
||||
await markTerminalSyncFailure(ctx, leased);
|
||||
await ctx.db.patch(leased._id, { status: 'failed', updatedAt: timestamp, finishedAt: timestamp, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: safeJobError() });
|
||||
return false;
|
||||
}
|
||||
await ctx.db.patch(leased._id, { status: 'queued', updatedAt: timestamp, dueAt: args.dueAt, leaseExpiresAt: undefined, leaseToken: undefined, workerId: undefined, lastError: 'Synchronization scheduled for retry', ...(args.cooldownGroup ? { cooldownGroup: args.cooldownGroup } : {}) });
|
||||
if (args.cooldownContext && args.cooldownUntil !== undefined && Number.isFinite(args.cooldownUntil)) {
|
||||
const contexts = [args.cooldownContext, ...(args.cooldownGroup ? [`group:${args.cooldownGroup}`] : [])];
|
||||
for (const context of contexts) {
|
||||
const existing = await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first();
|
||||
const until = Math.max(existing?.until ?? 0, args.cooldownUntil);
|
||||
if (existing) await ctx.db.patch(existing._id, { until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp });
|
||||
else await ctx.db.insert('syncCooldowns', { context, until, group: args.cooldownGroup, kind: 'server', updatedAt: timestamp });
|
||||
}
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
async function currentCooldown(ctx: any, context: string, group?: string): Promise<any | null> {
|
||||
const rows = [await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', context)).first()];
|
||||
if (group) rows.push(await ctx.db.query('syncCooldowns').withIndex('by_context', (q: any) => q.eq('context', `group:${group}`)).first());
|
||||
const row = rows.filter(Boolean).sort((left: any, right: any) => right.until - left.until)[0];
|
||||
return row && row.until > now() ? { until: row.until, group: row.group, kind: row.kind } : null;
|
||||
}
|
||||
|
||||
export const bridgeCooldown = queryGeneric({
|
||||
args: { ...bridgeArgs, context: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertBridge(args.bridgeSecret);
|
||||
return await currentCooldown(ctx, args.context);
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeSyncWorkerCooldown = queryGeneric({
|
||||
args: { ...workerBridgeArgs, context: v.string(), group: v.optional(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
return await currentCooldown(ctx, args.context, args.group);
|
||||
},
|
||||
});
|
||||
|
||||
async function markRefreshUncertainRecord(ctx: any, userId: string, characterId: number, lease: any, character: any, record: any): Promise<boolean> {
|
||||
if (!lease || !character || !record || record.version !== lease.expectedVersion) {
|
||||
if (lease) await ctx.db.delete(lease._id);
|
||||
return false;
|
||||
}
|
||||
const epoch = character.authorizationEpoch + 1;
|
||||
const statuses = DATASET_NAMES.map((dataset) => ({ dataset, status: 'blocked' as const, grantRevision: character.grantRevision, authorizationEpoch: epoch, credentialVersion: character.credentialVersion, invalidatedAt: now(), error: 'Reconnect required' }));
|
||||
// Retain the last complete generation metadata; authorization status blocks
|
||||
// reads until reconnect, while a later reconnect can explicitly replace it.
|
||||
await ctx.db.patch(character._id, { authorizationStatus: 'reconnect_required', authorizationEpoch: epoch, datasetStatuses: statuses, syncGeneration: undefined, syncLeaseExpiresAt: undefined, error: 'Reconnect required', state: 'error' });
|
||||
await ctx.db.delete(record._id); await ctx.db.delete(lease._id);
|
||||
await invalidateCharacterJobs(ctx, userId, characterId, now());
|
||||
return true;
|
||||
}
|
||||
|
||||
export const bridgeMarkRefreshDispatched = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false;
|
||||
await ctx.db.patch(lease._id, { dispatchStartedAt: now() });
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
/** Durable per-character refresh serialization. The encrypted grant never reaches browsers. */
|
||||
export const bridgeAcquireRefreshLease = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), ownerId: v.string(), leaseMs: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const character = await findCharacter(ctx, args.userId, args.characterId);
|
||||
if (!character || character.authorizationStatus !== 'active') throw new Error('Character authorization is not active');
|
||||
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!record) throw new Error('Character authorization is missing');
|
||||
const timestamp = now(); const leaseMs = args.leaseMs ?? 2 * 60 * 1000;
|
||||
if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) throw new Error('Invalid refresh lease duration');
|
||||
const existing = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (existing && existing.expiresAt > timestamp) return null;
|
||||
if (existing?.dispatchStartedAt !== undefined) {
|
||||
await markRefreshUncertainRecord(ctx, args.userId, args.characterId, existing, character, record);
|
||||
return null;
|
||||
}
|
||||
if (existing) await ctx.db.delete(existing._id);
|
||||
const leaseToken = newIdentifier('refresh');
|
||||
await ctx.db.insert('refreshLeases', { userId: args.userId, characterId: args.characterId, leaseToken, ownerId: args.ownerId, expiresAt: timestamp + leaseMs, expectedVersion: record.version });
|
||||
return { leaseToken, expiresAt: timestamp + leaseMs, expectedVersion: record.version, encryptedTokens: record.encryptedTokens, tokenExpiresAt: record.tokenExpiresAt, scopes: record.scopes, grantRevision: character.grantRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: character.credentialVersion };
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeRenewRefreshLease = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), leaseMs: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now()) return false;
|
||||
const leaseMs = args.leaseMs ?? 2 * 60 * 1000;
|
||||
if (!Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 15 * 60 * 1000) return false;
|
||||
await ctx.db.patch(lease._id, { expiresAt: now() + leaseMs }); return true;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeReleaseRefreshLease = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt !== undefined) return false;
|
||||
await ctx.db.delete(lease._id);
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeCommitRefresh = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number(), encryptedTokens: v.string(), tokenExpiresAt: v.number(), scopes: v.array(v.string()), jobId: v.optional(v.string()), jobLeaseToken: v.optional(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
const character = await findCharacter(ctx, args.userId, args.characterId);
|
||||
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId || lease.expiresAt <= now() || lease.dispatchStartedAt === undefined || !character || character.authorizationStatus !== 'active' || !record || record.version !== args.expectedVersion) return null;
|
||||
if (!validActualScopes(args.scopes) || !scopesSubset(args.scopes, character.scopes ?? [])) throw new Error('Invalid refreshed scopes');
|
||||
const nextVersion = record.version + 1;
|
||||
const nextRevision = character.grantRevision + (scopesEqual(character.scopes, args.scopes) ? 0 : 1);
|
||||
const updated = updateDatasetStatuses(character, args.scopes, nextRevision, character.authorizationEpoch, nextVersion, now());
|
||||
await ctx.db.patch(record._id, { encryptedTokens: args.encryptedTokens, tokenExpiresAt: args.tokenExpiresAt, scopes: args.scopes, version: nextVersion });
|
||||
await ctx.db.patch(character._id, { scopes: args.scopes, grantRevision: nextRevision, credentialVersion: nextVersion, datasetStatuses: updated.statuses, ...(updated.assetsInvalidated ? { state: 'error' as const, activeGeneration: undefined, syncedAt: undefined, expiresAt: undefined, syncGeneration: undefined, syncLeaseExpiresAt: undefined } : {}) });
|
||||
if (updated.assetsInvalidated) await invalidateCharacterJobs(ctx, args.userId, args.characterId, now());
|
||||
else if (args.jobId && args.jobLeaseToken) {
|
||||
const job = await findJob(ctx, args.jobId);
|
||||
if (job && job.userId === args.userId && job.characterId === args.characterId && job.status === 'running' && job.leaseToken === args.jobLeaseToken) {
|
||||
await ctx.db.patch(job._id, { grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, credentialVersion: nextVersion, scopes: args.scopes, updatedAt: now() });
|
||||
}
|
||||
}
|
||||
await ctx.db.delete(lease._id);
|
||||
return { saved: true, credentialVersion: nextVersion, grantRevision: nextRevision, authorizationEpoch: character.authorizationEpoch, scopes: args.scopes };
|
||||
},
|
||||
});
|
||||
|
||||
/** Unknown rotating-token outcomes fail closed and require an explicit reconnect. */
|
||||
export const bridgeMarkRefreshUncertain = mutationGeneric({
|
||||
args: { ...workerBridgeArgs, userId: v.string(), characterId: v.number(), leaseToken: v.string(), ownerId: v.string(), expectedVersion: v.number() },
|
||||
handler: async (ctx, args) => {
|
||||
assertSyncWorker(args.workerSecret);
|
||||
const lease = await ctx.db.query('refreshLeases').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (!lease || lease.leaseToken !== args.leaseToken || lease.ownerId !== args.ownerId) return false;
|
||||
const character = await findCharacter(ctx, args.userId, args.characterId);
|
||||
const record = await ctx.db.query('tokenRecords').withIndex('by_user_character', (q: any) => q.eq('userId', args.userId).eq('characterId', args.characterId)).first();
|
||||
if (lease.expectedVersion !== args.expectedVersion) return false;
|
||||
return await markRefreshUncertainRecord(ctx, args.userId, args.characterId, lease, character, record);
|
||||
},
|
||||
});
|
||||
|
||||
export const bridgeSaveInteraction = mutationGeneric({
|
||||
args: {
|
||||
...bridgeArgs,
|
||||
|
||||
+50
-1
@@ -4,6 +4,7 @@ import { v } from "convex/values";
|
||||
/** The persisted character state is deliberately a small, explicit state machine. */
|
||||
export const characterStateValidator = v.union(
|
||||
v.literal("never"),
|
||||
v.literal("queued"),
|
||||
v.literal("syncing"),
|
||||
v.literal("ready"),
|
||||
v.literal("error"),
|
||||
@@ -16,6 +17,7 @@ export const authorizationStatusValidator = v.union(
|
||||
v.literal("quarantined"),
|
||||
v.literal("revoked"),
|
||||
v.literal("transferred"),
|
||||
v.literal("reconnect_required"),
|
||||
);
|
||||
export const datasetStateValidator = v.union(
|
||||
v.literal("never"),
|
||||
@@ -180,9 +182,56 @@ export default defineSchema({
|
||||
updatedAt: v.number(),
|
||||
leaseExpiresAt: v.optional(v.number()),
|
||||
characterId: v.optional(v.number()),
|
||||
// Durable sync orchestration fields. They are optional for pre-T07
|
||||
// placeholder rows; new jobs always populate the complete fence.
|
||||
dataset: v.optional(v.string()),
|
||||
dedupeKey: v.optional(v.string()),
|
||||
dueAt: v.optional(v.number()),
|
||||
attempts: v.optional(v.number()),
|
||||
maxAttempts: v.optional(v.number()),
|
||||
priority: v.optional(v.number()),
|
||||
lastClaimedAt: v.optional(v.number()),
|
||||
leaseToken: v.optional(v.string()),
|
||||
workerId: v.optional(v.string()),
|
||||
cancelRequested: v.optional(v.boolean()),
|
||||
cancelledAt: v.optional(v.number()),
|
||||
startedAt: v.optional(v.number()),
|
||||
finishedAt: v.optional(v.number()),
|
||||
lastError: v.optional(v.string()),
|
||||
generation: v.optional(v.string()),
|
||||
grantRevision: v.optional(v.number()),
|
||||
authorizationEpoch: v.optional(v.number()),
|
||||
credentialVersion: v.optional(v.number()),
|
||||
scopes: v.optional(v.array(v.string())),
|
||||
ownerKind: v.optional(v.string()),
|
||||
authority: v.optional(v.string()),
|
||||
cooldownGroup: v.optional(v.string()),
|
||||
})
|
||||
.index("by_user_status", ["userId", "status"])
|
||||
.index("by_lease_expires_at", ["leaseExpiresAt"]),
|
||||
.index("by_lease_expires_at", ["leaseExpiresAt"])
|
||||
.index("by_dedupe", ["dedupeKey"])
|
||||
.index("by_status_due", ["status", "dueAt"])
|
||||
.index("by_kind_status_due", ["kind", "status", "dueAt"])
|
||||
.index("by_asset_sync_queue", ["kind", "dataset", "authority", "ownerKind", "status", "dueAt"])
|
||||
.index("by_user_asset_status", ["userId", "kind", "dataset", "authority", "ownerKind", "status"]),
|
||||
|
||||
refreshLeases: defineTable({
|
||||
userId: v.string(),
|
||||
characterId: v.number(),
|
||||
leaseToken: v.string(),
|
||||
ownerId: v.string(),
|
||||
expiresAt: v.number(),
|
||||
expectedVersion: v.number(),
|
||||
dispatchStartedAt: v.optional(v.number()),
|
||||
}).index("by_user_character", ["userId", "characterId"]),
|
||||
|
||||
syncCooldowns: defineTable({
|
||||
context: v.string(),
|
||||
until: v.number(),
|
||||
group: v.optional(v.string()),
|
||||
kind: v.string(),
|
||||
updatedAt: v.number(),
|
||||
}).index("by_context", ["context"]).index("by_until", ["until"]),
|
||||
|
||||
referenceSyncMetadata: defineTable({
|
||||
source: v.string(),
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined';
|
||||
export type SyncJob = {
|
||||
id: string;
|
||||
dataset: 'assets';
|
||||
characterId: number;
|
||||
status: SyncJobStatus;
|
||||
attempts: number;
|
||||
dueAt: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
leaseExpiresAt: number | null;
|
||||
cancelRequested: boolean;
|
||||
error?: string;
|
||||
};
|
||||
export type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' };
|
||||
export type SyncEnqueueResponse = { queued: number; jobs: SyncJob[]; skipped: SyncSkip[]; partial?: boolean; demo?: boolean };
|
||||
|
||||
const JOB_STATUSES = new Set<SyncJobStatus>(['queued', 'running', 'succeeded', 'failed', 'cancelled', 'quarantined']);
|
||||
const SKIP_REASONS = new Set<SyncSkip['reason']>(['authorization_required', 'queue_full', 'enqueue_failed']);
|
||||
|
||||
function parseJob(value: unknown): SyncJob | null {
|
||||
if (!value || typeof value !== 'object') return null;
|
||||
const row = value as Record<string, unknown>;
|
||||
if (typeof row.id !== 'string' || typeof row.characterId !== 'number' || !Number.isSafeInteger(row.characterId) || typeof row.status !== 'string' || !JOB_STATUSES.has(row.status as SyncJobStatus)) return null;
|
||||
return {
|
||||
id: row.id, dataset: 'assets', characterId: row.characterId, status: row.status as SyncJobStatus,
|
||||
attempts: typeof row.attempts === 'number' ? row.attempts : 0,
|
||||
dueAt: typeof row.dueAt === 'number' ? row.dueAt : 0,
|
||||
createdAt: typeof row.createdAt === 'number' ? row.createdAt : 0,
|
||||
updatedAt: typeof row.updatedAt === 'number' ? row.updatedAt : 0,
|
||||
leaseExpiresAt: typeof row.leaseExpiresAt === 'number' ? row.leaseExpiresAt : null,
|
||||
cancelRequested: row.cancelRequested === true,
|
||||
...(typeof row.error === 'string' ? { error: row.error } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseSyncEnqueueBody(value: unknown): SyncEnqueueResponse | { error: string } {
|
||||
if (!value || typeof value !== 'object') return { error: 'Synchronization response was invalid.' };
|
||||
const body = value as Record<string, unknown>;
|
||||
if (!Array.isArray(body.jobs) || !Array.isArray(body.skipped)) return { error: typeof body.error === 'string' ? body.error : 'Synchronization response was invalid.' };
|
||||
const jobs = body.jobs.map(parseJob);
|
||||
const skipped = body.skipped.map((entry: unknown) => {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
const row = entry as Record<string, unknown>;
|
||||
return typeof row.characterId === 'number' && Number.isSafeInteger(row.characterId) && typeof row.reason === 'string' && SKIP_REASONS.has(row.reason as SyncSkip['reason'])
|
||||
? { characterId: row.characterId, reason: row.reason as SyncSkip['reason'] } : null;
|
||||
});
|
||||
if (jobs.some((job) => job === null) || skipped.some((entry) => entry === null)) return { error: 'Synchronization response was invalid.' };
|
||||
const queued = typeof body.queued === 'number' && Number.isSafeInteger(body.queued) && body.queued >= 0 ? body.queued : jobs.length;
|
||||
return { queued, jobs: jobs as SyncJob[], skipped: skipped as SyncSkip[], ...(body.partial === true ? { partial: true } : {}), ...(body.demo === true ? { demo: true } : {}) };
|
||||
}
|
||||
|
||||
export async function enqueueSync(signal?: AbortSignal): Promise<{ response: Response; body: SyncEnqueueResponse | { error: string } }> {
|
||||
const response = await fetch('/api/sync', signal ? { method: 'POST', signal } : { method: 'POST' });
|
||||
return { response, body: parseSyncEnqueueBody(await response.json().catch(() => null)) };
|
||||
}
|
||||
|
||||
export const SYNC_STATUS_TIMEOUT_MS = 15_000;
|
||||
|
||||
export async function syncJobStatus(id: string, signal?: AbortSignal, timeoutMs = SYNC_STATUS_TIMEOUT_MS): Promise<SyncJob | null> {
|
||||
const requestController = new AbortController();
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const aborted = new Promise<never>((_, reject) => {
|
||||
onAbort = () => { requestController.abort(); reject(new DOMException('The operation was aborted', 'AbortError')); };
|
||||
if (signal?.aborted) onAbort();
|
||||
else signal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
const timedOut = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => { requestController.abort(); reject(new Error('Synchronization status timed out.')); }, timeoutMs);
|
||||
});
|
||||
try {
|
||||
const response = await Promise.race([fetch(`/api/sync/status?job=${encodeURIComponent(id)}`, { signal: requestController.signal }), timedOut, aborted]);
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw new Error('Synchronization status unavailable.');
|
||||
return await response.json() as SyncJob;
|
||||
} finally {
|
||||
if (timeout !== undefined) clearTimeout(timeout);
|
||||
if (signal && onAbort) signal.removeEventListener('abort', onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
export async function cancelSyncJob(id: string): Promise<SyncJob | null> {
|
||||
const response = await fetch('/api/sync/cancel', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ jobId: id }),
|
||||
});
|
||||
if (!response.ok) throw new Error('Could not cancel synchronization.');
|
||||
return await response.json() as SyncJob;
|
||||
}
|
||||
|
||||
export function syncTerminal(status: SyncJobStatus): boolean {
|
||||
return status === 'succeeded' || status === 'failed' || status === 'cancelled' || status === 'quarantined';
|
||||
}
|
||||
+16
-2
@@ -18,12 +18,12 @@ export type Character = {
|
||||
scopes: string[];
|
||||
/** Exactly one active primary may authenticate while signed out. */
|
||||
loginRole: 'primary' | 'data_only';
|
||||
authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred';
|
||||
authorizationStatus: 'active' | 'revocation_pending' | 'quarantined' | 'revoked' | 'transferred' | 'reconnect_required';
|
||||
grantRevision: number;
|
||||
credentialVersion: number;
|
||||
authorizationEpoch: number;
|
||||
datasets: CharacterDatasetStatus[];
|
||||
assets: { status: 'never' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string };
|
||||
assets: { status: 'never' | 'queued' | 'syncing' | 'ready' | 'error'; syncedAt?: number; expiresAt?: number; error?: string };
|
||||
};
|
||||
export type Asset = {
|
||||
itemId: string;
|
||||
@@ -307,6 +307,20 @@ export type JobOwner =
|
||||
|
||||
export type JobAuthority = 'user' | 'system';
|
||||
export type SyncJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'quarantined';
|
||||
export type SyncJobDataset = 'assets';
|
||||
export type SafeSyncJob = {
|
||||
id: string;
|
||||
dataset: SyncJobDataset;
|
||||
characterId: number;
|
||||
status: SyncJobStatus;
|
||||
attempts: number;
|
||||
dueAt: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
leaseExpiresAt: number | null;
|
||||
cancelRequested: boolean;
|
||||
error?: string;
|
||||
};
|
||||
export type PublicationOutcome = 'success' | 'failure';
|
||||
export type PublicationResourceFence = SelectedResource & {
|
||||
/** Monotonic revocation/quarantine epoch, distinct from grant and credential versions. */
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { RequestEvent } from '@sveltejs/kit';
|
||||
import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository } from '$lib/domain/model';
|
||||
import type { Asset, Character, CharacterAssetRepository, MarketEstimates, ReferenceRepository, SafeSyncJob } from '$lib/domain/model';
|
||||
import { AssetService } from '$lib/domain/assets';
|
||||
import { configuration, requireLiveConfig } from './config';
|
||||
import { ConvexStore } from './store';
|
||||
import { TokenCipher, hashSecret } from './security';
|
||||
import { EveSso, type VerifiedGrant } from './sso';
|
||||
import { EveSso } from './sso';
|
||||
import { EsiClient, EsiMarketEstimates } from './esi';
|
||||
import { SqliteReferenceRepository } from './reference';
|
||||
import { capabilityDisabledReason, capabilityEnabled, type ReleaseCapability } from './release-gate';
|
||||
@@ -91,104 +91,29 @@ export async function assetReport(
|
||||
finally { reference?.close(); }
|
||||
}
|
||||
|
||||
type CommittedGrant = VerifiedGrant & {
|
||||
credentialVersion: number;
|
||||
grantRevision: number;
|
||||
authorizationEpoch: number;
|
||||
};
|
||||
const refreshes = new Map<string, Promise<CommittedGrant>>();
|
||||
async function characterGrant(userId: string, characterId: number, revalidateOwnership = false): Promise<CommittedGrant> {
|
||||
const key = `${userId}:${characterId}`;
|
||||
const existing = refreshes.get(key); if (existing) return existing;
|
||||
const promise = (async () => {
|
||||
const { store, cipher, sso } = live();
|
||||
const record = await store.tokenRecord(userId, characterId);
|
||||
if (!record) throw new Error('Character authorization is missing');
|
||||
const previous = await cipher.open<VerifiedGrant>(record.encryptedTokens, `character:${characterId}`);
|
||||
if (previous.characterId !== characterId) throw new Error('Character authorization mismatch');
|
||||
if (!revalidateOwnership && previous.expiresAt > Date.now() + 60_000) {
|
||||
const current = (await store.characters(userId)).find((entry) => entry.id === characterId);
|
||||
if (!current) throw new Error('Character authorization is missing');
|
||||
return {
|
||||
...previous,
|
||||
credentialVersion: current.credentialVersion,
|
||||
grantRevision: current.grantRevision,
|
||||
authorizationEpoch: current.authorizationEpoch,
|
||||
};
|
||||
}
|
||||
const grant = await sso.refresh(previous.refreshToken);
|
||||
if (grant.characterId !== characterId || grant.ownerHash !== previous.ownerHash) {
|
||||
await store.quarantineAuthorization(userId, characterId).catch(() => {});
|
||||
throw new Error('Character ownership changed; reconnect required');
|
||||
}
|
||||
const saved = await store.saveTokens(userId, characterId, { encryptedTokens: await cipher.seal(grant, `character:${characterId}`), tokenExpiresAt: grant.expiresAt, scopes: grant.scopes, expectedVersion: record.version });
|
||||
if (!saved) throw new Error('Authorization changed concurrently; retry synchronization');
|
||||
return {
|
||||
...grant,
|
||||
credentialVersion: saved.credentialVersion,
|
||||
grantRevision: saved.grantRevision,
|
||||
authorizationEpoch: saved.authorizationEpoch,
|
||||
scopes: saved.scopes,
|
||||
};
|
||||
type SyncSkip = { characterId: number; reason: 'authorization_required' | 'queue_full' | 'enqueue_failed' };
|
||||
|
||||
})();
|
||||
refreshes.set(key, promise);
|
||||
try { return await promise; } finally { if (refreshes.get(key) === promise) refreshes.delete(key); }
|
||||
}
|
||||
export async function synchronize(userId: string) {
|
||||
export async function synchronize(userId: string, initiatingSessionHash?: string) {
|
||||
const config = configuration();
|
||||
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) {
|
||||
throw new Error(capabilityDisabledReason('asset_sync'));
|
||||
}
|
||||
if (config.mode === 'demo') return { completed: 2, failed: [] as number[], demo: true };
|
||||
const { store, esi } = live();
|
||||
if (config.mode === 'demo') return { queued: 2, jobs: [] as SafeSyncJob[], skipped: [] as SyncSkip[], partial: false, demo: true };
|
||||
if (!initiatingSessionHash) throw new Error('Authenticated session proof is required');
|
||||
const { store } = live();
|
||||
const characters = await store.characters(userId);
|
||||
const failed: number[] = []; let completed = 0;
|
||||
const jobs: SafeSyncJob[] = [];
|
||||
const skipped: SyncSkip[] = [];
|
||||
for (const character of characters) {
|
||||
if (character.authorizationStatus !== 'active') { failed.push(character.id); continue; }
|
||||
if (!character.scopes.includes('esi-assets.read_assets.v1')) { failed.push(character.id); continue; }
|
||||
let generation: string | undefined;
|
||||
if (character.authorizationStatus !== 'active' || !character.scopes.includes('esi-assets.read_assets.v1')) {
|
||||
skipped.push({ characterId: character.id, reason: 'authorization_required' });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// Refresh first so the generation fence records the credential version
|
||||
// that actually authorizes the ESI request and publication.
|
||||
const grant = await characterGrant(userId, character.id, true);
|
||||
if (!grant.scopes.includes('esi-assets.read_assets.v1')) throw new Error('Asset authorization missing');
|
||||
// Refresh/CAS may have advanced authority while the initial character
|
||||
// list was in memory. Reload before leasing and bind beginSync to the
|
||||
// exact revisions/scopes that authorize the transport request.
|
||||
const current = (await store.characters(userId)).find((entry) => entry.id === character.id);
|
||||
if (!current || current.authorizationStatus !== 'active'
|
||||
|| !current.scopes.includes('esi-assets.read_assets.v1')
|
||||
|| current.credentialVersion !== grant.credentialVersion
|
||||
|| current.grantRevision !== grant.grantRevision
|
||||
|| current.authorizationEpoch !== grant.authorizationEpoch
|
||||
|| JSON.stringify([...current.scopes].sort()) !== JSON.stringify([...grant.scopes].sort())) {
|
||||
throw new Error('Asset authorization changed after credential commit');
|
||||
}
|
||||
const authority = {
|
||||
userId,
|
||||
principal: grant.ownerHash,
|
||||
characterId: current.id,
|
||||
scopes: current.scopes,
|
||||
grantRevision: grant.grantRevision,
|
||||
authorizationEpoch: grant.authorizationEpoch,
|
||||
credentialVersion: grant.credentialVersion,
|
||||
} as const;
|
||||
generation = await store.beginSync(userId, current.id, {
|
||||
scopes: authority.scopes,
|
||||
grantRevision: authority.grantRevision,
|
||||
authorizationEpoch: authority.authorizationEpoch,
|
||||
credentialVersion: authority.credentialVersion,
|
||||
});
|
||||
const snapshot = await esi.getAssets(current.id, grant.accessToken, authority);
|
||||
const chunks = Math.ceil(snapshot.assets.length / 250);
|
||||
for (let index = 0; index < chunks; index++) await store.stageAssets(userId, character.id, generation, index, snapshot.assets.slice(index * 250, (index + 1) * 250));
|
||||
await store.finishSync(userId, character.id, generation, chunks, snapshot.syncedAt, snapshot.expiresAt);
|
||||
completed++;
|
||||
} catch {
|
||||
failed.push(character.id);
|
||||
if (generation) await store.failSync(userId, character.id, generation, 'Synchronization failed. Reconnect if authorization was revoked, or retry after ESI recovers.').catch(() => {});
|
||||
jobs.push(await store.enqueueAssetSync(userId, character.id, initiatingSessionHash));
|
||||
} catch (error) {
|
||||
skipped.push({ characterId: character.id, reason: error instanceof Error && error.message === 'Synchronization queue is full' ? 'queue_full' : 'enqueue_failed' });
|
||||
}
|
||||
}
|
||||
return { completed, failed, demo: false };
|
||||
return { queued: jobs.length, jobs, skipped, partial: skipped.length > 0, demo: false };
|
||||
}
|
||||
|
||||
+153
-7
@@ -5,6 +5,7 @@ import type {
|
||||
Character,
|
||||
CharacterAssetRepository,
|
||||
UserId,
|
||||
SafeSyncJob,
|
||||
} from "$lib/domain/model";
|
||||
|
||||
export type OAuthStateRecord = {
|
||||
@@ -67,6 +68,31 @@ export type SavedInteraction = SaveInteractionInput & {
|
||||
created: boolean;
|
||||
};
|
||||
|
||||
export type ClaimedSyncJob = SafeSyncJob & {
|
||||
jobId: string;
|
||||
leaseToken: string;
|
||||
userId: string;
|
||||
characterId: number;
|
||||
grantRevision: number;
|
||||
authorizationEpoch: number;
|
||||
credentialVersion: number;
|
||||
scopes: string[];
|
||||
generation?: string;
|
||||
cooldownGroup?: string;
|
||||
};
|
||||
|
||||
export type RefreshLease = {
|
||||
leaseToken: string;
|
||||
expiresAt: number;
|
||||
expectedVersion: number;
|
||||
encryptedTokens: string;
|
||||
tokenExpiresAt: number;
|
||||
scopes: string[];
|
||||
grantRevision: number;
|
||||
authorizationEpoch: number;
|
||||
credentialVersion: number;
|
||||
};
|
||||
|
||||
type AssetPage = {
|
||||
page: Asset[];
|
||||
isDone: boolean;
|
||||
@@ -83,6 +109,9 @@ const QUERY = {
|
||||
tokenRecord: "functions:bridgeTokenRecord",
|
||||
characters: "functions:bridgeCharacters",
|
||||
assetsPage: "functions:bridgeAssetsPage",
|
||||
syncJob: "functions:bridgeSyncJob",
|
||||
cooldown: "functions:bridgeCooldown",
|
||||
workerCooldown: "functions:bridgeSyncWorkerCooldown",
|
||||
} as const;
|
||||
|
||||
const MUTATION = {
|
||||
@@ -99,6 +128,23 @@ const MUTATION = {
|
||||
setPrimary: "functions:bridgeSetPrimary",
|
||||
invalidateAuthorization: "functions:bridgeInvalidateAuthorization",
|
||||
quarantineAuthorization: "functions:bridgeQuarantineAuthorization",
|
||||
enqueueAssetSync: "functions:bridgeEnqueueAssetSync",
|
||||
cancelSyncJob: "functions:bridgeCancelSyncJob",
|
||||
claimSyncJob: "functions:bridgeClaimSyncJob",
|
||||
renewSyncJob: "functions:bridgeRenewSyncJob",
|
||||
beginSyncJob: "functions:bridgeBeginSyncJob",
|
||||
refreshSyncJobAuthority: "functions:bridgeRefreshSyncJobAuthority",
|
||||
restartSyncJobGeneration: "functions:bridgeRestartSyncJobGeneration",
|
||||
stageSyncAssets: "functions:bridgeStageSyncAssets",
|
||||
commitSyncJob: "functions:bridgeCommitSyncJob",
|
||||
failSyncJob: "functions:bridgeFailSyncJob",
|
||||
deferSyncJob: "functions:bridgeDeferSyncJob",
|
||||
acquireRefreshLease: "functions:bridgeAcquireRefreshLease",
|
||||
renewRefreshLease: "functions:bridgeRenewRefreshLease",
|
||||
releaseRefreshLease: "functions:bridgeReleaseRefreshLease",
|
||||
commitRefresh: "functions:bridgeCommitRefresh",
|
||||
markRefreshUncertain: "functions:bridgeMarkRefreshUncertain",
|
||||
markRefreshDispatched: "functions:bridgeMarkRefreshDispatched",
|
||||
} as const;
|
||||
|
||||
function queryReference(name: string): any {
|
||||
@@ -110,28 +156,37 @@ function mutationReference(name: string): any {
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side Convex repository. Every call goes through a public bridge
|
||||
* function carrying the dedicated bridge secret; this adapter never uses a
|
||||
* deployment admin key and is not safe to import into browser code.
|
||||
* Server-side Convex repository. Web calls use the general bridge secret;
|
||||
* worker lifecycle calls use a separate narrowly-scoped worker secret. This
|
||||
* adapter never uses a deployment admin key and is not safe to import into
|
||||
* browser code.
|
||||
*/
|
||||
export class ConvexStore implements CharacterAssetRepository {
|
||||
private readonly client: ConvexHttpClient;
|
||||
private readonly bridgeSecret: string;
|
||||
private readonly bridgeSecret?: string;
|
||||
private readonly workerSecret?: string;
|
||||
|
||||
public constructor(url: string, bridgeSecret: string) {
|
||||
public constructor(url: string, bridgeSecret?: string, workerSecret?: string) {
|
||||
if (!url) throw new Error("Convex URL is required");
|
||||
if (!bridgeSecret) throw new Error("Convex bridge secret is required");
|
||||
if (!bridgeSecret && !workerSecret) throw new Error("Convex bridge or worker secret is required");
|
||||
this.bridgeSecret = bridgeSecret;
|
||||
this.workerSecret = workerSecret;
|
||||
this.client = new ConvexHttpClient(url, {
|
||||
skipConvexDeploymentUrlCheck: true,
|
||||
logger: false,
|
||||
});
|
||||
this.bridgeSecret = bridgeSecret;
|
||||
}
|
||||
|
||||
private bridgeArgs(args: Record<string, unknown>): Record<string, unknown> {
|
||||
if (!this.bridgeSecret) throw new Error("General bridge authority is unavailable to this store");
|
||||
return { ...args, bridgeSecret: this.bridgeSecret };
|
||||
}
|
||||
|
||||
private workerArgs(args: Record<string, unknown>): Record<string, unknown> {
|
||||
if (!this.workerSecret) throw new Error("Worker bridge authority is unavailable to this store");
|
||||
return { ...args, workerSecret: this.workerSecret };
|
||||
}
|
||||
|
||||
private async query<T>(name: string, args: Record<string, unknown>): Promise<T> {
|
||||
return (await this.client.query(
|
||||
queryReference(name),
|
||||
@@ -147,6 +202,21 @@ export class ConvexStore implements CharacterAssetRepository {
|
||||
)) as T;
|
||||
}
|
||||
|
||||
private async workerMutation<T>(name: string, args: Record<string, unknown>): Promise<T> {
|
||||
return (await this.client.mutation(
|
||||
mutationReference(name),
|
||||
this.workerArgs(args),
|
||||
{ skipQueue: true },
|
||||
)) as T;
|
||||
}
|
||||
|
||||
private async workerQuery<T>(name: string, args: Record<string, unknown>): Promise<T> {
|
||||
return (await this.client.query(
|
||||
queryReference(name),
|
||||
this.workerArgs(args),
|
||||
)) as T;
|
||||
}
|
||||
|
||||
public async session(hash: string): Promise<{ userId: string; sessionId: string } | null> {
|
||||
return await this.query<{ userId: string; sessionId: string } | null>(QUERY.session, {
|
||||
sessionHash: hash,
|
||||
@@ -294,6 +364,82 @@ export class ConvexStore implements CharacterAssetRepository {
|
||||
throw new Error(SNAPSHOT_CHANGED);
|
||||
}
|
||||
|
||||
public async enqueueAssetSync(userId: string, characterId: number, initiatingSessionHash: string): Promise<SafeSyncJob> {
|
||||
return await this.mutation<SafeSyncJob>(MUTATION.enqueueAssetSync, { userId, characterId, initiatingSessionHash });
|
||||
}
|
||||
|
||||
public async syncJob(userId: string, jobId: string): Promise<SafeSyncJob | null> {
|
||||
return await this.query<SafeSyncJob | null>(QUERY.syncJob, { userId, jobId });
|
||||
}
|
||||
|
||||
public async cancelSyncJob(userId: string, jobId: string, initiatingSessionHash: string): Promise<SafeSyncJob | null> {
|
||||
return await this.mutation<SafeSyncJob | null>(MUTATION.cancelSyncJob, { userId, jobId, initiatingSessionHash });
|
||||
}
|
||||
|
||||
public async claimSyncJob(workerId: string, leaseMs?: number): Promise<ClaimedSyncJob | null> {
|
||||
return await this.workerMutation<ClaimedSyncJob | null>(MUTATION.claimSyncJob, { workerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
|
||||
}
|
||||
|
||||
public async renewSyncJob(jobId: string, leaseToken: string, workerId: string, leaseMs?: number): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.renewSyncJob, { jobId, leaseToken, workerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
|
||||
}
|
||||
|
||||
public async beginSyncJob(jobId: string, leaseToken: string): Promise<string> {
|
||||
return await this.workerMutation<string>(MUTATION.beginSyncJob, { jobId, leaseToken });
|
||||
}
|
||||
|
||||
public async refreshSyncJobAuthority(jobId: string, leaseToken: string, expected: { grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }): Promise<{ grantRevision: number; authorizationEpoch: number; credentialVersion: number; scopes: string[] }> {
|
||||
return await this.workerMutation(MUTATION.refreshSyncJobAuthority, { jobId, leaseToken, expectedGrantRevision: expected.grantRevision, expectedAuthorizationEpoch: expected.authorizationEpoch, expectedCredentialVersion: expected.credentialVersion, expectedScopes: expected.scopes });
|
||||
}
|
||||
|
||||
public async restartSyncJobGeneration(jobId: string, leaseToken: string): Promise<void> {
|
||||
await this.workerMutation(MUTATION.restartSyncJobGeneration, { jobId, leaseToken });
|
||||
}
|
||||
|
||||
public async stageSyncAssets(jobId: string, leaseToken: string, index: number, assets: Asset[]): Promise<void> {
|
||||
await this.workerMutation(MUTATION.stageSyncAssets, { jobId, leaseToken, index, assets });
|
||||
}
|
||||
|
||||
public async commitSyncJob(jobId: string, leaseToken: string, chunkCount: number, syncedAt: number, expiresAt: number): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.commitSyncJob, { jobId, leaseToken, chunkCount, syncedAt, expiresAt });
|
||||
}
|
||||
|
||||
public async failSyncJob(jobId: string, leaseToken: string): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.failSyncJob, { jobId, leaseToken });
|
||||
}
|
||||
|
||||
public async deferSyncJob(jobId: string, leaseToken: string, dueAt: number, cooldown?: { context: string; until: number; group?: string }): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.deferSyncJob, { jobId, leaseToken, dueAt, ...(cooldown === undefined ? {} : { cooldownContext: cooldown.context, cooldownUntil: cooldown.until, ...(cooldown.group === undefined ? {} : { cooldownGroup: cooldown.group }) }) });
|
||||
}
|
||||
|
||||
public async cooldown(context: string, group?: string): Promise<{ until: number; group?: string; kind: string } | null> {
|
||||
return await this.workerQuery<{ until: number; group?: string; kind: string } | null>(QUERY.workerCooldown, { context, ...(group === undefined ? {} : { group }) });
|
||||
}
|
||||
|
||||
public async acquireRefreshLease(userId: string, characterId: number, ownerId: string, leaseMs?: number): Promise<RefreshLease | null> {
|
||||
return await this.workerMutation<RefreshLease | null>(MUTATION.acquireRefreshLease, { userId, characterId, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
|
||||
}
|
||||
|
||||
public async renewRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string, leaseMs?: number): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.renewRefreshLease, { userId, characterId, leaseToken, ownerId, ...(leaseMs === undefined ? {} : { leaseMs }) });
|
||||
}
|
||||
|
||||
public async releaseRefreshLease(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.releaseRefreshLease, { userId, characterId, leaseToken, ownerId });
|
||||
}
|
||||
|
||||
public async markRefreshDispatched(userId: string, characterId: number, leaseToken: string, ownerId: string): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.markRefreshDispatched, { userId, characterId, leaseToken, ownerId });
|
||||
}
|
||||
|
||||
public async commitRefresh(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number, input: { encryptedTokens: string; tokenExpiresAt: number; scopes: string[]; jobId?: string; jobLeaseToken?: string }): Promise<SavedTokenAuthority | null> {
|
||||
return await this.workerMutation<SavedTokenAuthority | null>(MUTATION.commitRefresh, { userId, characterId, leaseToken, ownerId, expectedVersion, ...input });
|
||||
}
|
||||
|
||||
public async markRefreshUncertain(userId: string, characterId: number, leaseToken: string, ownerId: string, expectedVersion: number): Promise<boolean> {
|
||||
return await this.workerMutation<boolean>(MUTATION.markRefreshUncertain, { userId, characterId, leaseToken, ownerId, expectedVersion });
|
||||
}
|
||||
|
||||
public async beginSync(
|
||||
userId: string,
|
||||
characterId: number,
|
||||
|
||||
+50
-11
@@ -1,6 +1,7 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { onDestroy, onMount } from 'svelte';
|
||||
import AssetResult from '$lib/components/AssetResult.svelte';
|
||||
import { cancelSyncJob, enqueueSync, syncJobStatus, syncTerminal, type SyncJob, type SyncSkip } from '$lib/client/sync-queue';
|
||||
import type { AssetOperation } from '$lib/domain/model';
|
||||
let { data } = $props();
|
||||
let prompt = $state('');
|
||||
@@ -10,29 +11,66 @@
|
||||
let message = $state('');
|
||||
let failure = $state('');
|
||||
let operation = $state<AssetOperation | null>(null);
|
||||
let syncJobs = $state<SyncJob[]>([]);
|
||||
let syncSkipped = $state<SyncSkip[]>([]);
|
||||
let pollController: AbortController | null = null;
|
||||
let enqueueController: AbortController | null = null;
|
||||
let requestController: AbortController | null = null;
|
||||
let pollTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
function stopPolling() { pollController?.abort(); pollController = null; if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; }
|
||||
async function pollSync(ids: string[]) {
|
||||
stopPolling();
|
||||
const controller = new AbortController(); pollController = controller;
|
||||
try {
|
||||
for (let attempt = 0; attempt < 120 && !controller.signal.aborted; attempt += 1) {
|
||||
const next = await Promise.all(ids.map((id) => syncJobStatus(id, controller.signal)));
|
||||
if (controller.signal.aborted) return;
|
||||
if (next.some((job) => job === null)) { busy = false; failure = 'A synchronization job disappeared before completion; no report retry was performed.'; return; }
|
||||
syncJobs = next as SyncJob[];
|
||||
if (syncJobs.every((job) => syncTerminal(job.status))) {
|
||||
busy = false;
|
||||
if (syncJobs.length > 0 && syncJobs.every((job) => job.status === 'succeeded') && syncSkipped.every((entry) => entry.reason === 'authorization_required')) await ask(submitted, true);
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
const done = () => { if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; controller.signal.removeEventListener('abort', done); resolve(); };
|
||||
pollTimer = setTimeout(done, 1_000); controller.signal.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
}
|
||||
if (!controller.signal.aborted) { busy = false; failure = 'Synchronization is still running; retry the report after the queued jobs finish.'; }
|
||||
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization status unavailable.'; } }
|
||||
finally { if (pollController === controller) pollController = null; }
|
||||
}
|
||||
onDestroy(() => { stopPolling(); enqueueController?.abort(); enqueueController = null; requestController?.abort(); requestController = null; });
|
||||
async function cancelSync(id: string) {
|
||||
try { const job = await cancelSyncJob(id); if (job) syncJobs = syncJobs.map((entry) => entry.id === job.id ? job : entry); }
|
||||
catch (error) { failure = error instanceof Error ? error.message : 'Could not cancel synchronization.'; }
|
||||
}
|
||||
async function ask(text = prompt, resume = false) {
|
||||
if (!text.trim() || busy) return;
|
||||
busy = true; failure = ''; submitted = text;
|
||||
requestController?.abort(); const controller = new AbortController(); requestController = controller;
|
||||
if (!resume || !requestId) requestId = crypto.randomUUID();
|
||||
sessionStorage.setItem('eve-pending', JSON.stringify({ text, requestId }));
|
||||
try {
|
||||
const response = await fetch('/api/chat', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: text, requestId }) });
|
||||
const response = await fetch('/api/chat', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: text, requestId }), signal: controller.signal });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Request failed');
|
||||
message = body.message; operation = body.operation ?? null;
|
||||
if (!operation || operation.status === 'ready') sessionStorage.removeItem('eve-pending');
|
||||
} catch (error) { failure = error instanceof Error ? error.message : 'Request failed'; }
|
||||
finally { busy = false; }
|
||||
} catch (error) { if (!controller.signal.aborted) failure = error instanceof Error ? error.message : 'Request failed'; }
|
||||
finally { if (requestController === controller) requestController = null; busy = false; }
|
||||
}
|
||||
async function sync() {
|
||||
busy = true; failure = '';
|
||||
stopPolling(); enqueueController?.abort(); const controller = new AbortController(); enqueueController = controller; busy = true; failure = '';
|
||||
try {
|
||||
const response = await fetch('/api/sync', { method: 'POST' });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Synchronization failed');
|
||||
busy = false; await ask(submitted, true);
|
||||
} catch (error) { failure = error instanceof Error ? error.message : 'Synchronization failed'; }
|
||||
finally { busy = false; }
|
||||
const { response, body } = await enqueueSync(controller.signal);
|
||||
if ('jobs' in body) { syncJobs = body.jobs; syncSkipped = body.skipped; }
|
||||
if (!response.ok || !('jobs' in body)) throw new Error(('error' in body && body.error) || (syncSkipped.some((entry) => entry.reason === 'queue_full') ? 'Synchronization queue is full.' : 'Synchronization could not be queued.'));
|
||||
if (syncJobs.length > 0) await pollSync(syncJobs.map((job) => job.id));
|
||||
else busy = false;
|
||||
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization failed'; } }
|
||||
finally { if (enqueueController === controller) enqueueController = null; }
|
||||
}
|
||||
onMount(() => {
|
||||
try {
|
||||
@@ -54,6 +92,7 @@
|
||||
</form>
|
||||
<div class="row"><button class="quiet" disabled={busy} onclick={() => ask('/value')}>Estimate all assets</button><button class="quiet" disabled={busy} onclick={() => ask('/assets')}>Browse assets</button></div>
|
||||
{#if failure}<p class="error" role="alert">{failure}</p>{/if}
|
||||
{#if syncJobs.length || syncSkipped.length}<section class="panel" aria-live="polite"><h2>Synchronization status</h2>{#each syncJobs as job}<p class="row spread"><span>Character {job.characterId}: {job.status}{job.error ? ` — ${job.error}` : ''}</span>{#if job.status === 'queued' || job.status === 'running'}<button class="quiet" onclick={() => cancelSync(job.id)}>Cancel</button>{/if}</p>{/each}{#each syncSkipped as entry}<p class="muted">Character {entry.characterId}: {entry.reason === 'authorization_required' ? 'authorization required' : entry.reason === 'queue_full' ? 'queue full' : 'enqueue failed'}</p>{/each}</section>{/if}
|
||||
<div aria-live="polite" aria-busy={busy}>
|
||||
{#if submitted}<p class="message">{submitted}</p>{/if}
|
||||
{#if message}<p>{message}</p>{/if}
|
||||
|
||||
@@ -4,6 +4,8 @@ import { session, synchronize } from '$lib/server/application';
|
||||
import { protectMutation } from '$lib/server/http';
|
||||
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
|
||||
import { configuration } from '$lib/server/config';
|
||||
import { hashSecret } from '$lib/server/security';
|
||||
import { SESSION_COOKIE } from '$lib/server/application';
|
||||
export const POST: RequestHandler = async (event) => {
|
||||
const config = configuration();
|
||||
// Bound origin/session work by the existing anonymous IP bucket first. The
|
||||
@@ -16,8 +18,11 @@ export const POST: RequestHandler = async (event) => {
|
||||
if (current) protectMutation(event, `sync:${current.userId}`, 6);
|
||||
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
|
||||
try {
|
||||
const result = await synchronize(current.userId);
|
||||
if (result.failed.length) return json({ ...result, error: `${result.failed.length} character(s) could not synchronize. Previous complete snapshots were preserved; check Characters.` }, { status: 503 });
|
||||
return json(result);
|
||||
const cookie = event.cookies.get(SESSION_COOKIE);
|
||||
if (!cookie) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
|
||||
const result = await synchronize(current.userId, hashSecret(cookie));
|
||||
const failures = result.skipped.filter((entry) => entry.reason !== 'authorization_required');
|
||||
const status = result.queued > 0 ? 202 : failures.some((entry) => entry.reason === 'queue_full') ? 429 : failures.length > 0 ? 503 : 200;
|
||||
return json(result, { status });
|
||||
} catch { return json({ error: 'Synchronization unavailable. Check backend configuration and try again.' }, { status: 503 }); }
|
||||
};
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { live, session, SESSION_COOKIE } from '$lib/server/application';
|
||||
import { hashSecret } from '$lib/server/security';
|
||||
import { configuration } from '$lib/server/config';
|
||||
import { boundedJson, protectMutation } from '$lib/server/http';
|
||||
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
|
||||
|
||||
export const POST: RequestHandler = async (event) => {
|
||||
const config = configuration();
|
||||
protectMutation(event, `sync-cancel:${event.getClientAddress()}`, 6);
|
||||
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) return json({ error: capabilityDisabledReason('asset_sync') }, { status: 503 });
|
||||
const current = await session(event);
|
||||
if (current) protectMutation(event, `sync-cancel:${current.userId}`, 6);
|
||||
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
|
||||
const body = await boundedJson(event) as { jobId?: unknown } | null;
|
||||
const jobId = body && typeof body.jobId === 'string' ? body.jobId : '';
|
||||
if (!jobId || jobId.length > 128) return json({ error: 'A sync job is required.' }, { status: 400 });
|
||||
const cookie = event.cookies.get(SESSION_COOKIE);
|
||||
if (!cookie) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
|
||||
try {
|
||||
const result = await live().store.cancelSyncJob(current.userId, jobId, hashSecret(cookie));
|
||||
return result ? json(result) : json({ error: 'Sync job not found.' }, { status: 404 });
|
||||
} catch {
|
||||
return json({ error: 'Synchronization cancellation unavailable.' }, { status: 503 });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { live, session } from '$lib/server/application';
|
||||
import { configuration } from '$lib/server/config';
|
||||
import { capabilityDisabledReason, capabilityEnabled } from '$lib/server/release-gate';
|
||||
|
||||
export const GET: RequestHandler = async (event) => {
|
||||
const config = configuration();
|
||||
if (!capabilityEnabled('asset_sync', config.capabilityAllowlist)) return json({ error: capabilityDisabledReason('asset_sync') }, { status: 503 });
|
||||
const current = await session(event);
|
||||
if (!current) return json({ error: 'Connect an EVE character first.' }, { status: 401 });
|
||||
const jobId = event.url.searchParams.get('job');
|
||||
if (!jobId || jobId.length > 128) return json({ error: 'A sync job is required.' }, { status: 400 });
|
||||
try {
|
||||
const job = await live().store.syncJob(current.userId, jobId);
|
||||
return job ? json(job) : json({ error: 'Sync job not found.' }, { status: 404 });
|
||||
} catch {
|
||||
return json({ error: 'Synchronization status unavailable.' }, { status: 503 });
|
||||
}
|
||||
};
|
||||
@@ -1,20 +1,60 @@
|
||||
<script lang="ts">
|
||||
import { onDestroy } from 'svelte';
|
||||
import { invalidateAll } from '$app/navigation';
|
||||
import AssetResult from '$lib/components/AssetResult.svelte';
|
||||
import { cancelSyncJob, enqueueSync, syncJobStatus, syncTerminal, type SyncJob, type SyncSkip } from '$lib/client/sync-queue';
|
||||
let { data } = $props();
|
||||
let busy = $state(false); let failure = $state('');
|
||||
async function sync() {
|
||||
busy = true; failure = '';
|
||||
let jobs = $state<SyncJob[]>([]); let skipped = $state<SyncSkip[]>([]);
|
||||
let pollController: AbortController | null = null;
|
||||
let enqueueController: AbortController | null = null;
|
||||
let pollTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
function stopPolling() { pollController?.abort(); pollController = null; if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; }
|
||||
async function poll(ids: string[]) {
|
||||
stopPolling();
|
||||
const controller = new AbortController(); pollController = controller;
|
||||
try {
|
||||
const response = await fetch('/api/sync', { method: 'POST' });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Synchronization failed');
|
||||
await invalidateAll();
|
||||
} catch (error) { failure = error instanceof Error ? error.message : 'Synchronization failed'; }
|
||||
finally { busy = false; }
|
||||
for (let attempt = 0; attempt < 120 && !controller.signal.aborted; attempt += 1) {
|
||||
const next = await Promise.all(ids.map((id) => syncJobStatus(id, controller.signal)));
|
||||
if (controller.signal.aborted) return;
|
||||
if (next.some((job) => job === null)) { busy = false; failure = 'A synchronization job disappeared before completion; no report retry was performed.'; return; }
|
||||
jobs = next as SyncJob[];
|
||||
if (jobs.every((job) => syncTerminal(job.status))) {
|
||||
busy = false;
|
||||
if (jobs.length > 0 && jobs.every((job) => job.status === 'succeeded') && skipped.every((entry) => entry.reason === 'authorization_required')) await invalidateAll();
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
const done = () => { if (pollTimer !== undefined) clearTimeout(pollTimer); pollTimer = undefined; controller.signal.removeEventListener('abort', done); resolve(); };
|
||||
pollTimer = setTimeout(done, 1_000); controller.signal.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
}
|
||||
if (!controller.signal.aborted) { busy = false; failure = 'Synchronization is still running; this page will retain its queued status.'; }
|
||||
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization status unavailable.'; } }
|
||||
finally { if (pollController === controller) pollController = null; }
|
||||
}
|
||||
onDestroy(() => { stopPolling(); enqueueController?.abort(); enqueueController = null; });
|
||||
async function sync() {
|
||||
stopPolling(); enqueueController?.abort(); const controller = new AbortController(); enqueueController = controller; busy = true; failure = '';
|
||||
try {
|
||||
const { response, body } = await enqueueSync(controller.signal);
|
||||
if ('jobs' in body) { jobs = body.jobs; skipped = body.skipped; }
|
||||
if (!response.ok || !('jobs' in body)) throw new Error(('error' in body && body.error) || (skipped.some((entry) => entry.reason === 'queue_full') ? 'Synchronization queue is full.' : 'Synchronization could not be queued.'));
|
||||
if (jobs.length > 0) await poll(jobs.map((job) => job.id));
|
||||
else busy = false;
|
||||
} catch (error) { if (!controller.signal.aborted) { busy = false; failure = error instanceof Error ? error.message : 'Synchronization failed'; } }
|
||||
finally { if (enqueueController === controller) enqueueController = null; }
|
||||
}
|
||||
async function cancel(id: string) {
|
||||
try { const job = await cancelSyncJob(id); if (job) jobs = jobs.map((entry) => entry.id === job.id ? job : entry); }
|
||||
catch (error) { failure = error instanceof Error ? error.message : 'Could not cancel synchronization.'; }
|
||||
}
|
||||
function skipLabel(reason: SyncSkip['reason']): string {
|
||||
return reason === 'authorization_required' ? 'authorization required' : reason === 'queue_full' ? 'queue full' : 'enqueue failed';
|
||||
}
|
||||
</script>
|
||||
<div class="row spread"><div><p class="eyebrow">SHARED DOMAIN VIEW</p><h1>Assets</h1></div><button class="quiet" disabled={busy} onclick={sync}>{busy ? 'Synchronizing…' : 'Synchronize characters'}</button></div>
|
||||
<p class="muted">All connected characters. The same deterministic report used by the assistant.</p>
|
||||
{#if failure}<p class="error" role="alert">{failure}</p>{/if}
|
||||
{#if jobs.length || skipped.length}<section class="panel" aria-live="polite"><h2>Synchronization status</h2>{#each jobs as job}<p class="row spread"><span>Character {job.characterId}: {job.status}{job.error ? ` — ${job.error}` : ''}</span>{#if job.status === 'queued' || job.status === 'running'}<button class="quiet" onclick={() => cancel(job.id)}>Cancel</button>{/if}</p>{/each}{#each skipped as entry}<p class="muted">Character {entry.characterId}: {skipLabel(entry.reason)}</p>{/each}</section>{/if}
|
||||
<AssetResult result={data.result} onSync={sync} />
|
||||
|
||||
Reference in New Issue
Block a user