checkpoint ISAC registries and system monitoring

src/lib/server/pagination.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import { getPagination, getRegistrySort } from './pagination.js';
+
+const options = {
+	name: { asc: 'name ASC', desc: 'name DESC' },
+	updated: { asc: 'updated_at ASC', desc: 'updated_at DESC', defaultDirection: 'desc' as const }
+};
+
+describe('getRegistrySort', () => {
+	it('returns only allowlisted sort expressions', () => {
+		const url = new URL('https://shd.build/isac/attributes?sort=name&direction=desc');
+		expect(getRegistrySort(url, options, 'name')).toEqual({
+			key: 'name',
+			direction: 'desc',
+			orderBy: 'name DESC'
+		});
+	});
+
+	it('falls back for unknown keys and directions', () => {
+		const url = new URL('https://shd.build/isac/attributes?sort=DROP%20TABLE&direction=sideways');
+		expect(getRegistrySort(url, options, 'updated')).toEqual({
+			key: 'updated',
+			direction: 'desc',
+			orderBy: 'updated_at DESC'
+		});
+	});
+
+	it('supports namespaced registry parameters', () => {
+		const url = new URL('https://shd.build/isac/users/1?loginSort=name&loginDirection=asc');
+		expect(getRegistrySort(url, options, 'updated', 'desc', 'login')).toMatchObject({
+			key: 'name',
+			direction: 'asc'
+		});
+	});
+});
+
+describe('getPagination', () => {
+	it('keeps supported row counts and clamps the selected page', () => {
+		const url = new URL('https://shd.build/isac/attributes?perPage=50&page=9');
+		expect(getPagination(url, 75)).toMatchObject({ perPage: 50, page: 2, pageCount: 2 });
+	});
+});

src/lib/server/pagination.ts

@@ -2,6 +2,39 @@ import type { URL } from 'node:url';
 
 export const PAGE_SIZE_PRESETS = [10, 25, 50, 100] as const;
 
+export type RegistrySortDirection = 'asc' | 'desc';
+export type RegistrySort = {
+	key: string;
+	direction: RegistrySortDirection;
+	orderBy: string;
+};
+export type RegistrySortOptions = Record<
+	string,
+	{ asc: string; desc: string; defaultDirection?: RegistrySortDirection }
+>;
+
+export function getRegistrySort(
+	url: URL,
+	options: RegistrySortOptions,
+	defaultKey: string,
+	defaultDirection: RegistrySortDirection = 'asc',
+	namespace = ''
+): RegistrySort {
+	const sortKey = namespace ? `${namespace}Sort` : 'sort';
+	const directionKey = namespace ? `${namespace}Direction` : 'direction';
+	const requestedKey = String(url.searchParams.get(sortKey) ?? '');
+	const key = Object.hasOwn(options, requestedKey) ? requestedKey : defaultKey;
+	const option = options[key];
+	if (!option) throw new Error(`Unknown default registry sort: ${defaultKey}`);
+	const requestedDirection = String(url.searchParams.get(directionKey) ?? '').toLowerCase();
+	const fallbackDirection = option.defaultDirection ?? defaultDirection;
+	const direction: RegistrySortDirection =
+		requestedDirection === 'asc' || requestedDirection === 'desc'
+			? requestedDirection
+			: fallbackDirection;
+	return { key, direction, orderBy: option[direction] };
+}
+
 export function getRegistrySearch(url: URL, namespace = '') {
 	const key = namespace ? `${namespace}Q` : 'q';
 	const search = String(url.searchParams.get(key) ?? '').normalize('NFKC').trim().slice(0, 80);

src/lib/server/remove-background.ts

@@ -1,4 +1,5 @@
 import sharp from 'sharp';
+import { recordRemoveBgCharge } from '$lib/server/remove-bg-monitor';
 
 const REMOVE_BG_ENDPOINT = 'https://api.remove.bg/v1.0/removebg';
 const REMOVE_BG_PREVIEW_EDGE = 1500;
@@ -41,6 +42,12 @@ export async function removeBackgroundPreview(input: Buffer): Promise<Buffer<Arr
 		throw new BackgroundRemovalError('remove.bg could not be reached. Try again or upload a transparent image.');
 	}
 	if (!response.ok) throw new BackgroundRemovalError(apiMessage(response.status));
+	const charged = Number(response.headers.get('X-Credits-Charged'));
+	if (Number.isFinite(charged) && charged > 0) {
+		await recordRemoveBgCharge(charged).catch((error) => {
+			console.error('Unable to update the cached remove.bg credit balance.', error);
+		});
+	}
 	const output = Buffer.from(await response.arrayBuffer());
 	const metadata = await sharp(output, { failOn: 'warning' }).metadata();
 	if (!metadata.hasAlpha) throw new BackgroundRemovalError('remove.bg returned an image without transparency.');

src/lib/server/remove-bg-monitor.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import { parseRemoveBgAccount } from './remove-bg-monitor';
+
+describe('parseRemoveBgAccount', () => {
+	it('normalizes the official account response fields', () => {
+		expect(parseRemoveBgAccount({
+			data: {
+				attributes: {
+					credits: {
+						total: 23.5,
+						subscription: 20,
+						payg: 3.5,
+						enterprise: 0
+					},
+					api: {
+						free_calls: 17,
+						sizes: 'preview'
+					}
+				}
+			}
+		})).toEqual({
+			total: 23.5,
+			subscription: 20,
+			payg: 3.5,
+			enterprise: 0,
+			freeCalls: 17
+		});
+	});
+
+	it('floors fractional free-call balances and accepts numeric API strings', () => {
+		expect(parseRemoveBgAccount({
+			data: {
+				attributes: {
+					credits: { total: '7.25', subscription: '5', payg: '2.25' },
+					api: { free_calls: '4.9' }
+				}
+			}
+		})).toEqual({
+			total: 7.25,
+			subscription: 5,
+			payg: 2.25,
+			enterprise: 0,
+			freeCalls: 4
+		});
+	});
+
+	it.each([
+		undefined,
+		{},
+		{ data: { attributes: {} } },
+		{ data: { attributes: { credits: { total: 'unknown' }, api: { free_calls: 10 } } } },
+		{ data: { attributes: { credits: { total: 10 }, api: { free_calls: 'unknown' } } } },
+		{ data: { attributes: { credits: { total: -1 }, api: { free_calls: 10 } } } },
+		{ data: { attributes: { credits: { total: 10 }, api: { free_calls: -1 } } } }
+	])('rejects a malformed or negative required balance: %j', (payload) => {
+		expect(() => parseRemoveBgAccount(payload)).toThrow('remove.bg returned an invalid account balance.');
+	});
+
+	it('defaults malformed or negative optional credit buckets to zero', () => {
+		expect(parseRemoveBgAccount({
+			data: {
+				attributes: {
+					credits: {
+						total: 12,
+						subscription: -1,
+						payg: 'not-a-number',
+						enterprise: Number.POSITIVE_INFINITY
+					},
+					api: { free_calls: 8 }
+				}
+			}
+		})).toEqual({
+			total: 12,
+			subscription: 0,
+			payg: 0,
+			enterprise: 0,
+			freeCalls: 8
+		});
+	});
+});

src/lib/server/remove-bg-monitor.ts

@@ -0,0 +1,236 @@
+import { randomUUID } from 'node:crypto';
+import { env } from '$env/dynamic/private';
+import { getDatabase } from '$lib/server/database';
+import { createNotification } from '$lib/server/notifications';
+
+const PROVIDER = 'remove_bg';
+const ACCOUNT_ENDPOINT = 'https://api.remove.bg/v1.0/account';
+
+export type RemoveBgCreditStatus = {
+	configured: boolean;
+	total: number | null;
+	subscription: number | null;
+	payg: number | null;
+	enterprise: number | null;
+	freeCalls: number | null;
+	checkedAt: string | null;
+	nextCheckAt: string | null;
+	lastCharge: number | null;
+	lastChargeAt: string | null;
+	error: string | null;
+	severity: 'unknown' | 'ok' | 'warning' | 'critical';
+	warningCreditThreshold: number;
+	warningFreeCallThreshold: number;
+};
+
+const finiteNonNegative = (value: unknown) => {
+	const parsed = Number(value);
+	return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
+};
+
+export function parseRemoveBgAccount(payload: unknown) {
+	const attributes = (payload as { data?: { attributes?: Record<string, unknown> } })?.data?.attributes;
+	const credits = attributes?.credits as Record<string, unknown> | undefined;
+	const api = attributes?.api as Record<string, unknown> | undefined;
+	const total = finiteNonNegative(credits?.total);
+	const freeCalls = finiteNonNegative(api?.free_calls);
+	if (total === null || freeCalls === null) throw new Error('remove.bg returned an invalid account balance.');
+	return {
+		total,
+		subscription: finiteNonNegative(credits?.subscription) ?? 0,
+		payg: finiteNonNegative(credits?.payg) ?? 0,
+		enterprise: finiteNonNegative(credits?.enterprise) ?? 0,
+		freeCalls: Math.floor(freeCalls)
+	};
+}
+
+function settings() {
+	const warningCreditThreshold = finiteNonNegative(env.REMOVE_BG_LOW_CREDIT_THRESHOLD) ?? 5;
+	const warningFreeCallThreshold = finiteNonNegative(env.REMOVE_BG_LOW_FREE_CALL_THRESHOLD) ?? 10;
+	const checkHours = Math.max(1, finiteNonNegative(env.REMOVE_BG_CREDIT_CHECK_HOURS) ?? 6);
+	return { warningCreditThreshold, warningFreeCallThreshold, checkHours };
+}
+
+function severity(total: number, freeCalls: number) {
+	const { warningCreditThreshold, warningFreeCallThreshold } = settings();
+	if (total <= 0 && freeCalls <= 0) return 'critical' as const;
+	if (total <= warningCreditThreshold && freeCalls <= warningFreeCallThreshold) return 'warning' as const;
+	return 'ok' as const;
+}
+
+async function activeAdministrators() {
+	const database = await getDatabase();
+	return (await database.prepare(`
+		SELECT DISTINCT users.id
+		FROM users
+		LEFT JOIN deleted_accounts deleted ON deleted.user_id=users.id
+		LEFT JOIN user_status status ON status.user_id=users.id
+		LEFT JOIN user_roles ON user_roles.user_id=users.id
+		LEFT JOIN role_permissions ON role_permissions.role_id=user_roles.role_id
+		LEFT JOIN permissions role_permission ON role_permission.id=role_permissions.permission_id
+		LEFT JOIN user_permissions ON user_permissions.user_id=users.id
+		LEFT JOIN permissions direct_permission ON direct_permission.id=user_permissions.permission_id
+		WHERE deleted.user_id IS NULL AND status.blocked_at IS NULL
+			AND (status.banned_until IS NULL OR status.banned_until<=?)
+			AND (users.role='admin' OR user_roles.role_id='role-admin'
+				OR role_permission.name='system.admin' OR direct_permission.name='system.admin')
+	`)).all(new Date().toISOString());
+}
+
+async function queueAlertEpisode(episodeId: string, alertSeverity: 'warning' | 'critical') {
+	const database = await getDatabase();
+	const now = new Date().toISOString();
+	for (const administrator of await activeAdministrators()) {
+		await (await database.prepare(`INSERT OR IGNORE INTO external_service_alert_deliveries
+			(id,provider,episode_id,severity,user_id,created_at)
+			VALUES(?,?,?,?,?,?)`)).run(
+			randomUUID(), PROVIDER, episodeId, alertSeverity, administrator.id, now
+		);
+	}
+}
+
+async function persistBalance(balance: ReturnType<typeof parseRemoveBgAccount>) {
+	const database = await getDatabase();
+	const previous = await (await database.prepare(
+		'SELECT severity,episode_id FROM external_service_status WHERE provider=?'
+	)).get(PROVIDER);
+	const now = new Date();
+	const nextCheck = new Date(now.getTime() + settings().checkHours * 60 * 60_000);
+	const nextSeverity = severity(balance.total, balance.freeCalls);
+	const previousSeverity = String(previous?.severity ?? 'unknown');
+	const recovered = nextSeverity === 'ok' && (previousSeverity === 'warning' || previousSeverity === 'critical');
+	const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
+		(previousSeverity === 'unknown' || previousSeverity === 'ok');
+	const episodeId = startsEpisode ? randomUUID() : String(previous?.episode_id ?? '') || null;
+	await (await database.prepare(`INSERT INTO external_service_status(
+		provider,credits_total,credits_subscription,credits_payg,credits_enterprise,free_calls,
+		checked_at,next_check_at,error_message,severity,episode_id,episode_started_at,recovered_at,updated_at
+	) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+	ON CONFLICT(provider) DO UPDATE SET
+		credits_total=excluded.credits_total,credits_subscription=excluded.credits_subscription,
+		credits_payg=excluded.credits_payg,credits_enterprise=excluded.credits_enterprise,
+		free_calls=excluded.free_calls,checked_at=excluded.checked_at,next_check_at=excluded.next_check_at,
+		error_message=NULL,severity=excluded.severity,episode_id=excluded.episode_id,
+		episode_started_at=CASE WHEN excluded.episode_id<>external_service_status.episode_id THEN excluded.episode_started_at ELSE external_service_status.episode_started_at END,
+		recovered_at=excluded.recovered_at,updated_at=excluded.updated_at`)).run(
+		PROVIDER, balance.total, balance.subscription, balance.payg, balance.enterprise, balance.freeCalls,
+		now.toISOString(), nextCheck.toISOString(), null, nextSeverity,
+		recovered ? null : episodeId, startsEpisode ? now.toISOString() : null,
+		recovered ? now.toISOString() : null, now.toISOString()
+	);
+	if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
+		await queueAlertEpisode(episodeId, nextSeverity);
+	}
+}
+
+export async function refreshRemoveBgCredits(force = false) {
+	const database = await getDatabase();
+	const existing = await (await database.prepare(
+		'SELECT next_check_at FROM external_service_status WHERE provider=?'
+	)).get(PROVIDER);
+	if (!force && existing?.next_check_at && String(existing.next_check_at) > new Date().toISOString()) {
+		return getRemoveBgCreditStatus();
+	}
+	const apiKey = env.REMOVE_BG_API_KEY?.trim();
+	if (!apiKey) return getRemoveBgCreditStatus();
+	try {
+		const response = await fetch(ACCOUNT_ENDPOINT, {
+			headers: { 'X-Api-Key': apiKey },
+			signal: AbortSignal.timeout(15_000)
+		});
+		if (!response.ok) throw new Error(`remove.bg account check failed with status ${response.status}.`);
+		await persistBalance(parseRemoveBgAccount(await response.json()));
+	} catch (error) {
+		const now = new Date();
+		const retryAt = new Date(now.getTime() + 60 * 60_000);
+		const message = error instanceof Error ? error.message : 'remove.bg account check failed.';
+		await (await database.prepare(`INSERT INTO external_service_status(provider,next_check_at,error_message,updated_at)
+			VALUES(?,?,?,?) ON CONFLICT(provider) DO UPDATE SET
+			next_check_at=excluded.next_check_at,error_message=excluded.error_message,updated_at=excluded.updated_at`))
+			.run(PROVIDER, retryAt.toISOString(), message.slice(0, 240), now.toISOString());
+	}
+	return getRemoveBgCreditStatus();
+}
+
+export async function recordRemoveBgCharge(charged: number) {
+	if (!Number.isFinite(charged) || charged <= 0) return;
+	const database = await getDatabase();
+	const row = await (await database.prepare(
+		'SELECT credits_total,free_calls,severity,episode_id FROM external_service_status WHERE provider=?'
+	)).get(PROVIDER);
+	if (row?.credits_total === null || row?.credits_total === undefined) return;
+	const now = new Date().toISOString();
+	const total = Math.max(0, Number(row.credits_total) - charged);
+	const nextSeverity = severity(total, Number(row.free_calls ?? 0));
+	const previousSeverity = String(row.severity ?? 'unknown');
+	const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
+		(previousSeverity === 'unknown' || previousSeverity === 'ok');
+	const episodeId = startsEpisode ? randomUUID() : String(row.episode_id ?? '') || null;
+	await (await database.prepare(`UPDATE external_service_status SET credits_total=?,last_charge=?,last_charge_at=?,
+		severity=?,episode_id=?,episode_started_at=CASE WHEN ? THEN ? ELSE episode_started_at END,updated_at=?
+		WHERE provider=?`)).run(
+		total, charged, now, nextSeverity, episodeId, startsEpisode ? 1 : 0, now, now, PROVIDER
+	);
+	if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
+		await queueAlertEpisode(episodeId, nextSeverity);
+	}
+}
+
+export async function deliverRemoveBgCreditAlerts() {
+	const database = await getDatabase();
+	const pending = await (await database.prepare(`SELECT delivery.*,status.credits_total,status.free_calls
+		FROM external_service_alert_deliveries delivery
+		JOIN external_service_status status ON status.provider=delivery.provider
+		WHERE delivery.delivered_at IS NULL ORDER BY delivery.created_at LIMIT 100`)).all();
+	let delivered = 0;
+	for (const alert of pending) {
+		const title = alert.severity === 'critical'
+			? 'remove.bg credits exhausted'
+			: 'remove.bg credits running low';
+		const body = `remove.bg has ${Number(alert.credits_total ?? 0).toLocaleString()} paid credits and ${Number(alert.free_calls ?? 0).toLocaleString()} free calls remaining. Review image-processing capacity in System Metrics.`;
+		const notificationId = await createNotification({
+			userId: String(alert.user_id),
+			category: 'isac',
+			source: 'ISAC SYSTEM OVERSIGHT',
+			title,
+			body,
+			link: '/isac/metrics',
+			actions: [{ label: 'VIEW SYSTEM METRICS', href: '/isac/metrics' }],
+			mandatory: true
+		});
+		await (await database.prepare(`UPDATE external_service_alert_deliveries
+			SET notification_id=?,delivered_at=? WHERE id=? AND delivered_at IS NULL`))
+			.run(notificationId, new Date().toISOString(), alert.id);
+		delivered++;
+	}
+	return delivered;
+}
+
+export async function getRemoveBgCreditStatus(): Promise<RemoveBgCreditStatus> {
+	const database = await getDatabase();
+	const row = await (await database.prepare('SELECT * FROM external_service_status WHERE provider=?')).get(PROVIDER);
+	const { warningCreditThreshold, warningFreeCallThreshold } = settings();
+	return {
+		configured: Boolean(env.REMOVE_BG_API_KEY?.trim()),
+		total: finiteNonNegative(row?.credits_total),
+		subscription: finiteNonNegative(row?.credits_subscription),
+		payg: finiteNonNegative(row?.credits_payg),
+		enterprise: finiteNonNegative(row?.credits_enterprise),
+		freeCalls: finiteNonNegative(row?.free_calls),
+		checkedAt: row?.checked_at ? String(row.checked_at) : null,
+		nextCheckAt: row?.next_check_at ? String(row.next_check_at) : null,
+		lastCharge: finiteNonNegative(row?.last_charge),
+		lastChargeAt: row?.last_charge_at ? String(row.last_charge_at) : null,
+		error: row?.error_message ? String(row.error_message) : null,
+		severity: (['ok', 'warning', 'critical'].includes(String(row?.severity))
+			? String(row?.severity)
+			: 'unknown') as RemoveBgCreditStatus['severity'],
+		warningCreditThreshold,
+		warningFreeCallThreshold
+	};
+}
+
+export async function runRemoveBgCreditMonitor() {
+	await refreshRemoveBgCredits(false);
+	return deliverRemoveBgCreditAlerts();
+}

src/lib/server/scheduled-notifications.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { env } from '$env/dynamic/private';
 import { getDatabase } from '$lib/server/database';
 import nodemailer from 'nodemailer';
+import { runRemoveBgCreditMonitor } from '$lib/server/remove-bg-monitor';
 
 function mailTransport() {
 	if (!env.SMTP_HOST || !env.SMTP_FROM) return null;
@@ -159,14 +160,15 @@ async function deliverDailyDigests() {
 	return sent;
 }
 
-let running: Promise<{ system: number; oversight: number; digests: number }> | null = null;
+let running: Promise<{ system: number; oversight: number; digests: number; serviceAlerts: number }> | null = null;
 export function runScheduledNotificationTasks() {
 	if (running) return running;
 	running = (async () => {
 		const system = await deliverSystemMessages();
 		const oversight = await deliverOversightMessages();
 		const digests = await deliverDailyDigests();
-		return { system, oversight, digests };
+		const serviceAlerts = await runRemoveBgCreditMonitor();
+		return { system, oversight, digests, serviceAlerts };
 	})()
 		.finally(() => { running = null; });
 	return running;