jb/eden
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add durable asset synchronization worker

Browse Source
This commit is contained in:
jb
2026-09-19 22:15:23 -04:00
parent 2675ea8114
commit fe7d218195
36 changed files with 2218 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/configure-convex.ts
+2 -1
View File
@@ -1,7 +1,8 @@
export {};
// Run only against your intended self-hosted deployment. Values are never printed.
const requiredNames = ['CONVEX_BRIDGE_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const;
const requiredNames = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_ISSUER', 'APP_AUTH_JWKS', 'APP_CAPABILITY_ALLOWLIST'] as const;
if (!process.env.CONVEX_SELF_HOSTED_URL || !process.env.CONVEX_SELF_HOSTED_ADMIN_KEY) throw new Error('Set self-hosted URL and admin key before configuring Convex.');
if (process.env.CONVEX_SYNC_WORKER_SECRET && process.env.CONVEX_SYNC_WORKER_SECRET === process.env.CONVEX_BRIDGE_SECRET) throw new Error('Worker and general bridge secrets must be distinct.');
for (const name of requiredNames) {
const value = process.env[name];
if (value === undefined || (name !== 'APP_CAPABILITY_ALLOWLIST' && !value)) {
scripts/generate-keys.ts
+2 -1
View File
@@ -6,8 +6,9 @@ const publicJwk = { ...await exportJWK(publicKey), kid: 'app-v1', alg: 'RS256',
const values = {
TOKEN_ENCRYPTION_KEY: randomBytes(32).toString('base64'),
CONVEX_BRIDGE_SECRET: randomBytes(32).toString('base64url'),
CONVEX_SYNC_WORKER_SECRET: randomBytes(32).toString('base64url'),
APP_AUTH_PRIVATE_KEY: Buffer.from(await exportPKCS8(privateKey)).toString('base64'),
APP_AUTH_JWKS: `data:application/json;base64,${Buffer.from(JSON.stringify({ keys: [publicJwk] })).toString('base64')}`
};
await writeFile('.env.keys', Object.entries(values).map(([k, v]) => `${k}='${v}'`).join('\n') + '\n', { flag: 'wx', mode: 0o600 });
console.log('Wrote .env.keys (0600). Import its four values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.');
console.log('Wrote .env.keys (0600). Import its five values into your Infisical project (dev, /), or a private .env for standalone setup. Never commit it. Existing files are not overwritten.');
scripts/lib/infisical.ts
+8 -7
View File
@@ -12,9 +12,11 @@ export interface VaultProject {
export const BOOTSTRAP_KEYS = ['INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET'] as const;
export const APPLICATION_KEYS = [
'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TYPESAFE_API_KEY', 'TOKEN_ENCRYPTION_KEY',
'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS',
'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS',
] as const;
export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const;
export const WORKER_APPLICATION_KEYS = ['EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_SYNC_WORKER_SECRET'] as const;
export const CONFIGURE_APPLICATION_KEYS = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_JWKS'] as const;
export const REQUIRED_KEYS = ['EVE_CLIENT_ID', 'TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS'] as const;
export const ADMIN_KEY = 'CONVEX_SELF_HOSTED_ADMIN_KEY';
export type Environment = Record<string, string | undefined>;
export type CliRunner = (args: string[], env: Environment) => Promise<string>;
@@ -157,7 +159,7 @@ const PUBLIC_KEYS = new Set([
'WEB_BIND_ADDRESS', 'CONVEX_BIND_ADDRESS', 'WEB_PORT', 'CONVEX_PORT', 'CONVEX_SITE_PORT',
'CONVEX_IMAGE', 'CONVEX_DASHBOARD_IMAGE', 'RELEASE_ENV',
]);
export function applicationEnvironment(base: Environment, secrets: Record<string, string>, publicConfig: Environment, deployment = false): Environment {
export function applicationEnvironment(base: Environment, secrets: Record<string, string>, publicConfig: Environment, deployment = false, target: 'web' | 'worker' | 'configure' = 'web'): Environment {
const env = cleanEnvironment(base);
// Public file is the configured deployment, not defaults inherited from Bun's .env.local.
for (const [key, value] of Object.entries(publicConfig)) {
@@ -165,10 +167,9 @@ export function applicationEnvironment(base: Environment, secrets: Record<string
env[key] = value;
}
// Managed app keys must come from the selected vault, never a stale ambient .env.
for (const key of APPLICATION_KEYS) {
delete env[key];
if (secrets[key] !== undefined) env[key] = secrets[key];
}
for (const key of APPLICATION_KEYS) delete env[key];
const allowedKeys = target === 'worker' ? WORKER_APPLICATION_KEYS : target === 'configure' ? CONFIGURE_APPLICATION_KEYS : APPLICATION_KEYS.filter((key) => key !== 'CONVEX_SYNC_WORKER_SECRET');
for (const key of allowedKeys) if (secrets[key] !== undefined) env[key] = secrets[key];
if (deployment && secrets[ADMIN_KEY]) env[ADMIN_KEY] = secrets[ADMIN_KEY];
return env;
}
scripts/release-check.ts
+1 -1
View File
@@ -65,7 +65,7 @@ function trackedFiles(): string[] {
return result.stdout.split('\n').filter(Boolean);
}
const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token'];
const MANAGED_SECRET_NAMES = ['TOKEN_ENCRYPTION_KEY', 'CONVEX_BRIDGE_SECRET', 'CONVEX_SYNC_WORKER_SECRET', 'APP_AUTH_PRIVATE_KEY', 'TYPESAFE_API_KEY', 'refresh_token'];
function browserBundleFiles(directory = 'build/client'): string[] {
try { statSync(directory); } catch { return []; }
const files: string[] = [];
scripts/run-sync-worker.ts
+42
View File
@@ -0,0 +1,42 @@
import { existsSync } from 'node:fs';
import { rm } from 'node:fs/promises';
import { join } from 'node:path';
const forbidden = ['CONVEX_BRIDGE_SECRET', 'CONVEX_SELF_HOSTED_ADMIN_KEY', 'APP_AUTH_PRIVATE_KEY', 'APP_AUTH_JWKS', 'TYPESAFE_API_KEY'];
const workerEnvironmentKeys = [
'PATH', 'HOME', 'TMPDIR', 'NODE_ENV', 'LANG', 'LC_ALL', 'TZ', 'NODE_EXTRA_CA_CERTS',
'CONVEX_URL', 'CONVEX_SYNC_WORKER_SECRET', 'TOKEN_ENCRYPTION_KEY',
'EVE_CLIENT_ID', 'EVE_CLIENT_SECRET', 'EVE_CALLBACK_URL', 'ESI_USER_AGENT',
'SYNC_WORKER_ID', 'SYNC_WORKER_LEASE_MS', 'SYNC_WORKER_HEARTBEAT_MS', 'SYNC_WORKER_POLL_MS', 'SYNC_WORKER_STARTUP_CHECK',
] as const;
export function workerEnvironment(base: Record<string, string | undefined>): Record<string, string> {
return Object.fromEntries(workerEnvironmentKeys.flatMap((key) => base[key] === undefined ? [] : [[key, base[key] as string]]));
}
export async function runSyncWorkerLauncher(): Promise<void> {
for (const name of forbidden) {
if (process.env[name]) throw new Error(`Sync worker refuses forbidden runtime credential: ${name}`);
}
if (!process.env.CONVEX_SYNC_WORKER_SECRET) throw new Error('CONVEX_SYNC_WORKER_SECRET is required for the sync worker');
const bundled = join(process.cwd(), 'build', 'sync-worker.js');
let entry = bundled;
let temporary = false;
if (!existsSync(entry)) {
entry = join('/tmp', `eve-sync-worker-${process.pid}.js`);
const result = await Bun.build({ entrypoints: ['workers/sync.ts'], target: 'bun', outdir: '/tmp', naming: `eve-sync-worker-${process.pid}.js`, minify: false });
if (!result.success) throw new Error('Could not bundle the sync worker');
temporary = true;
}
const child = Bun.spawn(['bun', entry], { env: workerEnvironment(process.env), stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
const interrupt = () => child.kill('SIGINT');
const terminate = () => child.kill('SIGTERM');
process.on('SIGINT', interrupt); process.on('SIGTERM', terminate);
try { process.exitCode = await child.exited; }
finally {
process.off('SIGINT', interrupt); process.off('SIGTERM', terminate);
if (temporary) await rm(entry, { force: true });
}
}
if (import.meta.main) await runSyncWorkerLauncher();
scripts/secrets.ts
+11 -4
View File
@@ -10,7 +10,7 @@ const HELP = `Infisical-backed development deployment (secret values are never p
bun run secrets:run -- <trusted command> [arguments...]
bun run secrets:run --deployment -- <trusted deployment command> [arguments...]
Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env)
Optional: --credentials-file PATH (default ~/.config/evebot/infisical.env), --worker to inject only worker-safe application secrets, or --deployment --configure for Convex function configuration.
Project metadata: .infisical.json. Public deployment configuration: config/dev.env.
Only allowlisted app secrets are injected. --deployment additionally permits the Convex admin key
and, when absent from the vault, derives it from the local persistent backend without printing it.
@@ -23,22 +23,29 @@ export function parseArguments(args: string[]) {
let command: string[] = [];
let credentialsFile = join(homedir(), '.config/evebot/infisical.env');
let deployment = false;
let worker = false;
let configure = false;
for (let i = 0; i < rest.length; i++) {
if (rest[i] === '--') { command = rest.slice(i + 1); break; }
if (rest[i] === '--credentials-file' && rest[i + 1] && !rest[i + 1].startsWith('--')) credentialsFile = resolve(rest[++i]);
else if (rest[i] === '--deployment') deployment = true;
else if (rest[i] === '--worker') worker = true;
else if (rest[i] === '--configure') configure = true;
// `bun run` consumes its own -- separator before forwarding script arguments.
else if (action === 'run' && !rest[i].startsWith('-')) { command = rest.slice(i); break; }
else throw new Error('Unknown or incomplete secrets helper option');
}
if (action === 'run' && !command.length) throw new Error('Supply a trusted command after --');
if (action === 'status' && command.length) throw new Error('status does not execute a command');
return { action, command, credentialsFile, deployment };
if (worker && deployment) throw new Error('Worker secret mode cannot request deployment admin credentials');
if (worker && configure) throw new Error('Worker and configure secret modes are mutually exclusive');
if (configure && !deployment) throw new Error('--configure requires --deployment so the Convex admin credential is available');
return { action, command, credentialsFile, deployment, worker, configure };
}
async function main(args: string[]): Promise<void> {
if (!args.length || args[0] === '--help') { console.log(HELP); return; }
const { action, command, credentialsFile, deployment } = parseArguments(args);
const { action, command, credentialsFile, deployment, worker, configure } = parseArguments(args);
const project = validateProject(JSON.parse(await readFile(new URL('../.infisical.json', import.meta.url), 'utf8')));
const credentials = await readCredentials(credentialsFile);
const secrets = await fetchSecrets(project, credentials);
@@ -54,7 +61,7 @@ async function main(args: string[]): Promise<void> {
return;
}
const publicConfig = parseEnv(await readFile(new URL('../config/dev.env', import.meta.url), 'utf8'));
const env = applicationEnvironment(process.env, secrets, publicConfig, deployment);
const env = applicationEnvironment(process.env, secrets, publicConfig, deployment, worker ? 'worker' : configure ? 'configure' : 'web');
if (deployment && !env[ADMIN_KEY]) env[ADMIN_KEY] = await localDeploymentAdmin(env);
const child = Bun.spawn(command, { env, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
const interrupt = () => child.kill('SIGINT');