From f5eb2fd6630aeb9050cfe8ac09068b5ddc3e6ba4 Mon Sep 17 00:00:00 2001 From: JB Date: Sun, 26 Jul 2026 10:38:07 -0400 Subject: [PATCH] Run scheduled tasks through application database --- .env.example | 3 + .env.staging | 1 + deploy/compose.staging.yml | 7 +- deploy/container-entrypoint.sh | 7 +- docs/DEPLOYMENT.md | 7 +- scripts/scheduler-client.mjs | 17 ++ src/lib/server/scheduled-notifications.ts | 173 +++++++++++++++++++ src/routes/api/internal/scheduler/+server.ts | 20 +++ 8 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 scripts/scheduler-client.mjs create mode 100644 src/lib/server/scheduled-notifications.ts create mode 100644 src/routes/api/internal/scheduler/+server.ts diff --git a/.env.example b/.env.example index 56d888b..4761f18 100644 --- a/.env.example +++ b/.env.example @@ -37,3 +37,6 @@ SMTP_SECURE=false SMTP_USER= SMTP_PASSWORD= SMTP_FROM=SHD.build + +# Long random bearer token used only by the internal container scheduler. +SCHEDULER_TOKEN= diff --git a/.env.staging b/.env.staging index d10de01..9cf2544 100644 --- a/.env.staging +++ b/.env.staging @@ -25,3 +25,4 @@ SMTP_PASSWORD="encrypted:BNsDbxMZqE142HZ8PyEytL9mG7L2y79rMvM6aqDcs0ONNYtjM2ffDwK SMTP_FROM="encrypted:BDsOlqmQFyAGOOJ2f+h9AS+CmV3uxEpy1mJaNzWIGFI1NMumy9s+dFgvKrunhAIImDSzod5BphAAOtUW8RyrIEO5wJvB1IJ+qoAw06hSWa4qWu1thdSu2ZEP3D+4ll7kpsOcMEU5PVFlSHQRLsPRdOS5mSGCy5VkXb2zhNAQ0A==" ORIGIN="encrypted:BHOwfV6ahbNXg8A7HNbApjaDrQiwVXSI5f/tBx/eeFYdBxgQJ8zQQKUOzSeICdOtx1z9BxIOWHr86+E8ZG67k0RxSeevpSLWe8bINOy+7Xf9IzXBgg2aOpY0gWxAcXercLKvP+UNJJy3TuEl9sQZqRJBjNPyhyJB" +SCHEDULER_TOKEN="encrypted:BOQwJdKcGoxWA70HS/7Rpv6uQiupCw+QuY54At7zuDZFopdUCOj+lu2N9I8L+497bROGBsWG2knGzu4NXvb264PS6c5kk3UGrcctAPMalN4Cb9mAm/ZHeC0rvTkF0Lk50/2sesc9Zmi8Qess8+hSRdyAttt8V7YoT6b4kDsX6sK1ws+s6cd+IG7Oxfc=" diff --git a/deploy/compose.staging.yml b/deploy/compose.staging.yml index cf13075..0d1b327 100644 --- a/deploy/compose.staging.yml +++ b/deploy/compose.staging.yml @@ -40,12 +40,7 @@ services: env_file: - .env.staging.runtime environment: - SKIP_DEFAULT_USERS: "true" - TDR_DATABASE_PATH: /app/data/shd-build-staging.db - TDR_AVATAR_STORAGE_PATH: /app/data/avatars - TDR_GAME_ICON_STORAGE_PATH: /app/data/game-icons - volumes: - - shd_stage_data:/app/data + SCHEDULER_URL: http://app:3000/api/internal/scheduler depends_on: app: condition: service_healthy diff --git a/deploy/container-entrypoint.sh b/deploy/container-entrypoint.sh index c11c63b..15c5d7f 100644 --- a/deploy/container-entrypoint.sh +++ b/deploy/container-entrypoint.sh @@ -7,12 +7,7 @@ if [ "${1:-}" = "serve" ]; then fi if [ "${1:-}" = "scheduler" ]; then - while true; do - node scripts/system-notification-dispatch.mjs - node scripts/oversight-notification-dispatch.mjs - node scripts/notification-digest.mjs - sleep 60 - done + exec node scripts/scheduler-client.mjs fi exec "$@" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b2c2766..2f4209c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -109,8 +109,11 @@ The workflow: The short outage is limited to backup, migration, and container startup. This follows the database-preservation policy while minimizing data-entry downtime. -The scheduler runs automated system messages, Oversight consolidation, and -email digest checks once per minute. +The scheduler calls a bearer-authenticated endpoint on the application service +once per minute. Scheduled message delivery therefore uses the web process's +existing embedded-database connection instead of opening the locked database +from a second process. The endpoint runs automated system messages, Oversight +consolidation, and email digest checks and is not published through a host port. ## Initial development-data bootstrap diff --git a/scripts/scheduler-client.mjs b/scripts/scheduler-client.mjs new file mode 100644 index 0000000..38fa1c8 --- /dev/null +++ b/scripts/scheduler-client.mjs @@ -0,0 +1,17 @@ +const endpoint = process.env.SCHEDULER_URL; +const token = process.env.SCHEDULER_TOKEN; +if (!endpoint || !token) throw new Error('SCHEDULER_URL and SCHEDULER_TOKEN are required.'); + +while (true) { + try { + const response = await fetch(endpoint, { + method: 'POST', + headers: { authorization: `Bearer ${token}` } + }); + if (!response.ok) throw new Error(`Scheduler endpoint returned HTTP ${response.status}.`); + console.log(`Scheduler tick: ${JSON.stringify(await response.json())}`); + } catch (error) { + console.error('Scheduler tick failed', error); + } + await new Promise((resolve) => setTimeout(resolve, 60_000)); +} diff --git a/src/lib/server/scheduled-notifications.ts b/src/lib/server/scheduled-notifications.ts new file mode 100644 index 0000000..cd11cf0 --- /dev/null +++ b/src/lib/server/scheduled-notifications.ts @@ -0,0 +1,173 @@ +import { randomUUID } from 'node:crypto'; +import { env } from '$env/dynamic/private'; +import { getDatabase } from '$lib/server/database'; +import nodemailer from 'nodemailer'; + +function mailTransport() { + if (!env.SMTP_HOST || !env.SMTP_FROM) return null; + return nodemailer.createTransport({ + host: env.SMTP_HOST, + port: Number(env.SMTP_PORT || 587), + secure: env.SMTP_SECURE === 'true', + auth: env.SMTP_USER ? { user: env.SMTP_USER, pass: env.SMTP_PASSWORD } : undefined, + requireTLS: env.SMTP_SECURE !== 'true', + disableFileAccess: true, + disableUrlAccess: true + }); +} + +const escapeHtml = (value: unknown) => String(value).replace(/[&<>"']/g, (character) => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' +})[character] ?? character); + +async function deliverSystemMessages() { + const database = await getDatabase(); + const now = new Date().toISOString(); + const due = await (await database.prepare(`SELECT delivery.id AS delivery_id,delivery.user_id,message.* + FROM automated_system_message_deliveries delivery + JOIN automated_system_messages message ON message.id=delivery.message_id + WHERE delivery.delivered_at IS NULL AND delivery.scheduled_for<=? AND message.enabled=1 + ORDER BY delivery.scheduled_for LIMIT 250`)).all(now); + const transport = mailTransport(); + const origin = (env.APP_ORIGIN || 'http://localhost:5173').replace(/\/+$/, ''); + let delivered = 0; + for (const message of due) { + const notificationId = randomUUID(); + await (await database.prepare(`INSERT INTO notifications + (id,user_id,category,source,title,body,link,actions_json,mandatory,created_at) + VALUES(?,?,'isac',?,?,?,?,?,1,?)`)).run( + notificationId, message.user_id, message.source, message.title, message.body, + message.link ?? null, message.actions_json ?? null, now + ); + await (await database.prepare(`UPDATE automated_system_message_deliveries + SET delivered_at=?,notification_id=? WHERE id=? AND delivered_at IS NULL`)) + .run(now, notificationId, message.delivery_id); + const preference = await (await database.prepare( + "SELECT email_frequency FROM notification_preferences WHERE user_id=? AND category='isac'" + )).get(message.user_id); + if (transport && preference?.email_frequency === 'immediate') { + const identity = await (await database.prepare('SELECT email FROM user_emails WHERE user_id=?')).get(message.user_id); + if (typeof identity?.email === 'string') { + try { + await transport.sendMail({ + from: env.SMTP_FROM, + to: identity.email, + subject: `SHD.build · ${message.title}`, + text: `${message.body}\n\n${message.link ? `${origin}${message.link}` : origin}`, + html: `

${escapeHtml(message.source)}

${escapeHtml(message.title)}

${escapeHtml(message.body)}

${message.link ? `OPEN SHD.BUILD` : ''}
` + }); + await (await database.prepare('UPDATE notifications SET emailed_at=? WHERE id=?')).run(now, notificationId); + } catch (error) { + console.error(`Unable to email automated notification ${notificationId}`, error); + } + } + } + delivered++; + } + return delivered; +} + +async function deliverOversightMessages() { + const database = await getDatabase(); + const now = new Date().toISOString(); + const due = await (await database.prepare(`SELECT * FROM oversight_notification_queue + WHERE delivered_at IS NULL AND deliver_after<=? ORDER BY user_id,oversight_category,created_at`)).all(now); + const groups = new Map[]>(); + for (const item of due) { + const key = `${item.user_id}:${item.oversight_category}:${item.frequency}:${item.deliver_after}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key)?.push(item); + } + const transport = mailTransport(); + const origin = (env.APP_ORIGIN || 'http://localhost:5173').replace(/\/+$/, ''); + let delivered = 0; + for (const items of groups.values()) { + const first = items[0]; + const categoryLabel = first.oversight_category === 'avatars' ? 'avatar' : 'data change'; + const title = `${items.length} ${categoryLabel} review${items.length === 1 ? '' : 's'} awaiting Oversight`; + const details = items.slice(0, 10).map((item) => `• ${item.summary}`).join('\n'); + const body = `${details}${items.length > 10 ? `\n• ${items.length - 10} additional review items` : ''}`; + const notificationId = randomUUID(); + await (await database.prepare(`INSERT INTO notifications + (id,user_id,category,source,title,body,link,mandatory,created_at) + VALUES(?,?,'isac','ISAC OVERSIGHT NETWORK',?,?,?,1,?)`)).run( + notificationId, first.user_id, title, body, first.link, now + ); + for (const item of items) { + await (await database.prepare( + 'UPDATE oversight_notification_queue SET delivered_at=? WHERE id=? AND delivered_at IS NULL' + )).run(now, item.id); + } + const preference = await (await database.prepare( + "SELECT email_frequency FROM notification_preferences WHERE user_id=? AND category='isac'" + )).get(first.user_id); + if (transport && preference?.email_frequency === 'immediate') { + const identity = await (await database.prepare('SELECT email FROM user_emails WHERE user_id=?')).get(first.user_id); + if (typeof identity?.email === 'string') { + try { + await transport.sendMail({ + from: env.SMTP_FROM, + to: identity.email, + subject: `SHD.build · ${title}`, + text: `${body}\n\n${origin}${first.link}`, + html: `

ISAC OVERSIGHT NETWORK

${escapeHtml(title)}

${escapeHtml(body)}

OPEN OVERSIGHT QUEUE
` + }); + await (await database.prepare('UPDATE notifications SET emailed_at=? WHERE id=?')).run(now, notificationId); + } catch (error) { + console.error(`Unable to email Oversight digest ${notificationId}`, error); + } + } + } + delivered += items.length; + } + return delivered; +} + +async function deliverDailyDigests() { + const transport = mailTransport(); + if (!transport) return 0; + const database = await getDatabase(); + const origin = (env.APP_ORIGIN || 'http://localhost:5173').replace(/\/+$/, ''); + const recipients = await (await database.prepare(`SELECT DISTINCT n.user_id,e.email + FROM notifications n + JOIN notification_preferences p ON p.user_id=n.user_id AND p.category=n.category AND p.email_frequency='daily' + JOIN user_emails e ON e.user_id=n.user_id + WHERE n.emailed_at IS NULL AND n.created_at>=p.updated_at`)).all(); + let sent = 0; + for (const recipient of recipients) { + const items = await (await database.prepare(`SELECT n.id,n.title,n.body,n.link FROM notifications n + JOIN notification_preferences p ON p.user_id=n.user_id AND p.category=n.category AND p.email_frequency='daily' + WHERE n.user_id=? AND n.emailed_at IS NULL AND n.created_at>=p.updated_at ORDER BY n.created_at LIMIT 100` + )).all(recipient.user_id); + if (!items.length) continue; + const rows = items.map((item) => `
${escapeHtml(item.title)}

${escapeHtml(item.body)}

${item.link ? `VIEW INTELLIGENCE` : ''}
`).join(''); + await transport.sendMail({ + from: env.SMTP_FROM, + to: String(recipient.email), + subject: `SHD.build daily notification summary · ${items.length} update${items.length === 1 ? '' : 's'}`, + text: items.map((item) => `${item.title}\n${item.body}\n${item.link ? `${origin}${item.link}` : ''}`).join('\n\n'), + html: `

SHD DAILY INTELLIGENCE

${items.length} notification${items.length === 1 ? '' : 's'}

${rows}
` + }); + const now = new Date().toISOString(); + for (const item of items) { + await (await database.prepare( + 'UPDATE notifications SET emailed_at=? WHERE id=? AND emailed_at IS NULL' + )).run(now, item.id); + } + sent++; + } + return sent; +} + +let running: Promise<{ system: number; oversight: number; digests: number }> | null = null; +export function runScheduledNotificationTasks() { + if (running) return running; + running = (async () => { + const system = await deliverSystemMessages(); + const oversight = await deliverOversightMessages(); + const digests = await deliverDailyDigests(); + return { system, oversight, digests }; + })() + .finally(() => { running = null; }); + return running; +} diff --git a/src/routes/api/internal/scheduler/+server.ts b/src/routes/api/internal/scheduler/+server.ts new file mode 100644 index 0000000..057e0a8 --- /dev/null +++ b/src/routes/api/internal/scheduler/+server.ts @@ -0,0 +1,20 @@ +import { createHash, timingSafeEqual } from 'node:crypto'; +import { env } from '$env/dynamic/private'; +import { runScheduledNotificationTasks } from '$lib/server/scheduled-notifications'; +import { json } from '@sveltejs/kit'; + +function tokenMatches(header: string | null) { + const configured = env.SCHEDULER_TOKEN; + const provided = header?.startsWith('Bearer ') ? header.slice(7) : ''; + if (!configured || !provided) return false; + const expectedHash = createHash('sha256').update(configured).digest(); + const providedHash = createHash('sha256').update(provided).digest(); + return timingSafeEqual(expectedHash, providedHash); +} + +export async function POST({ request }) { + if (!tokenMatches(request.headers.get('authorization'))) { + return json({ error: 'Not found' }, { status: 404 }); + } + return json({ status: 'ok', ...(await runScheduledNotificationTasks()) }); +}