checkpoint ISAC registries and system monitoring
This commit is contained in:
JB
@@ -0,0 +1,236 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import { createNotification } from '$lib/server/notifications';
|
||||
|
||||
const PROVIDER = 'remove_bg';
|
||||
const ACCOUNT_ENDPOINT = 'https://api.remove.bg/v1.0/account';
|
||||
|
||||
export type RemoveBgCreditStatus = {
|
||||
configured: boolean;
|
||||
total: number | null;
|
||||
subscription: number | null;
|
||||
payg: number | null;
|
||||
enterprise: number | null;
|
||||
freeCalls: number | null;
|
||||
checkedAt: string | null;
|
||||
nextCheckAt: string | null;
|
||||
lastCharge: number | null;
|
||||
lastChargeAt: string | null;
|
||||
error: string | null;
|
||||
severity: 'unknown' | 'ok' | 'warning' | 'critical';
|
||||
warningCreditThreshold: number;
|
||||
warningFreeCallThreshold: number;
|
||||
};
|
||||
|
||||
const finiteNonNegative = (value: unknown) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
||||
};
|
||||
|
||||
export function parseRemoveBgAccount(payload: unknown) {
|
||||
const attributes = (payload as { data?: { attributes?: Record<string, unknown> } })?.data?.attributes;
|
||||
const credits = attributes?.credits as Record<string, unknown> | undefined;
|
||||
const api = attributes?.api as Record<string, unknown> | undefined;
|
||||
const total = finiteNonNegative(credits?.total);
|
||||
const freeCalls = finiteNonNegative(api?.free_calls);
|
||||
if (total === null || freeCalls === null) throw new Error('remove.bg returned an invalid account balance.');
|
||||
return {
|
||||
total,
|
||||
subscription: finiteNonNegative(credits?.subscription) ?? 0,
|
||||
payg: finiteNonNegative(credits?.payg) ?? 0,
|
||||
enterprise: finiteNonNegative(credits?.enterprise) ?? 0,
|
||||
freeCalls: Math.floor(freeCalls)
|
||||
};
|
||||
}
|
||||
|
||||
function settings() {
|
||||
const warningCreditThreshold = finiteNonNegative(env.REMOVE_BG_LOW_CREDIT_THRESHOLD) ?? 5;
|
||||
const warningFreeCallThreshold = finiteNonNegative(env.REMOVE_BG_LOW_FREE_CALL_THRESHOLD) ?? 10;
|
||||
const checkHours = Math.max(1, finiteNonNegative(env.REMOVE_BG_CREDIT_CHECK_HOURS) ?? 6);
|
||||
return { warningCreditThreshold, warningFreeCallThreshold, checkHours };
|
||||
}
|
||||
|
||||
function severity(total: number, freeCalls: number) {
|
||||
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
||||
if (total <= 0 && freeCalls <= 0) return 'critical' as const;
|
||||
if (total <= warningCreditThreshold && freeCalls <= warningFreeCallThreshold) return 'warning' as const;
|
||||
return 'ok' as const;
|
||||
}
|
||||
|
||||
async function activeAdministrators() {
|
||||
const database = await getDatabase();
|
||||
return (await database.prepare(`
|
||||
SELECT DISTINCT users.id
|
||||
FROM users
|
||||
LEFT JOIN deleted_accounts deleted ON deleted.user_id=users.id
|
||||
LEFT JOIN user_status status ON status.user_id=users.id
|
||||
LEFT JOIN user_roles ON user_roles.user_id=users.id
|
||||
LEFT JOIN role_permissions ON role_permissions.role_id=user_roles.role_id
|
||||
LEFT JOIN permissions role_permission ON role_permission.id=role_permissions.permission_id
|
||||
LEFT JOIN user_permissions ON user_permissions.user_id=users.id
|
||||
LEFT JOIN permissions direct_permission ON direct_permission.id=user_permissions.permission_id
|
||||
WHERE deleted.user_id IS NULL AND status.blocked_at IS NULL
|
||||
AND (status.banned_until IS NULL OR status.banned_until<=?)
|
||||
AND (users.role='admin' OR user_roles.role_id='role-admin'
|
||||
OR role_permission.name='system.admin' OR direct_permission.name='system.admin')
|
||||
`)).all(new Date().toISOString());
|
||||
}
|
||||
|
||||
async function queueAlertEpisode(episodeId: string, alertSeverity: 'warning' | 'critical') {
|
||||
const database = await getDatabase();
|
||||
const now = new Date().toISOString();
|
||||
for (const administrator of await activeAdministrators()) {
|
||||
await (await database.prepare(`INSERT OR IGNORE INTO external_service_alert_deliveries
|
||||
(id,provider,episode_id,severity,user_id,created_at)
|
||||
VALUES(?,?,?,?,?,?)`)).run(
|
||||
randomUUID(), PROVIDER, episodeId, alertSeverity, administrator.id, now
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function persistBalance(balance: ReturnType<typeof parseRemoveBgAccount>) {
|
||||
const database = await getDatabase();
|
||||
const previous = await (await database.prepare(
|
||||
'SELECT severity,episode_id FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
const now = new Date();
|
||||
const nextCheck = new Date(now.getTime() + settings().checkHours * 60 * 60_000);
|
||||
const nextSeverity = severity(balance.total, balance.freeCalls);
|
||||
const previousSeverity = String(previous?.severity ?? 'unknown');
|
||||
const recovered = nextSeverity === 'ok' && (previousSeverity === 'warning' || previousSeverity === 'critical');
|
||||
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
||||
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
||||
const episodeId = startsEpisode ? randomUUID() : String(previous?.episode_id ?? '') || null;
|
||||
await (await database.prepare(`INSERT INTO external_service_status(
|
||||
provider,credits_total,credits_subscription,credits_payg,credits_enterprise,free_calls,
|
||||
checked_at,next_check_at,error_message,severity,episode_id,episode_started_at,recovered_at,updated_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(provider) DO UPDATE SET
|
||||
credits_total=excluded.credits_total,credits_subscription=excluded.credits_subscription,
|
||||
credits_payg=excluded.credits_payg,credits_enterprise=excluded.credits_enterprise,
|
||||
free_calls=excluded.free_calls,checked_at=excluded.checked_at,next_check_at=excluded.next_check_at,
|
||||
error_message=NULL,severity=excluded.severity,episode_id=excluded.episode_id,
|
||||
episode_started_at=CASE WHEN excluded.episode_id<>external_service_status.episode_id THEN excluded.episode_started_at ELSE external_service_status.episode_started_at END,
|
||||
recovered_at=excluded.recovered_at,updated_at=excluded.updated_at`)).run(
|
||||
PROVIDER, balance.total, balance.subscription, balance.payg, balance.enterprise, balance.freeCalls,
|
||||
now.toISOString(), nextCheck.toISOString(), null, nextSeverity,
|
||||
recovered ? null : episodeId, startsEpisode ? now.toISOString() : null,
|
||||
recovered ? now.toISOString() : null, now.toISOString()
|
||||
);
|
||||
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
||||
await queueAlertEpisode(episodeId, nextSeverity);
|
||||
}
|
||||
}
|
||||
|
||||
export async function refreshRemoveBgCredits(force = false) {
|
||||
const database = await getDatabase();
|
||||
const existing = await (await database.prepare(
|
||||
'SELECT next_check_at FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
if (!force && existing?.next_check_at && String(existing.next_check_at) > new Date().toISOString()) {
|
||||
return getRemoveBgCreditStatus();
|
||||
}
|
||||
const apiKey = env.REMOVE_BG_API_KEY?.trim();
|
||||
if (!apiKey) return getRemoveBgCreditStatus();
|
||||
try {
|
||||
const response = await fetch(ACCOUNT_ENDPOINT, {
|
||||
headers: { 'X-Api-Key': apiKey },
|
||||
signal: AbortSignal.timeout(15_000)
|
||||
});
|
||||
if (!response.ok) throw new Error(`remove.bg account check failed with status ${response.status}.`);
|
||||
await persistBalance(parseRemoveBgAccount(await response.json()));
|
||||
} catch (error) {
|
||||
const now = new Date();
|
||||
const retryAt = new Date(now.getTime() + 60 * 60_000);
|
||||
const message = error instanceof Error ? error.message : 'remove.bg account check failed.';
|
||||
await (await database.prepare(`INSERT INTO external_service_status(provider,next_check_at,error_message,updated_at)
|
||||
VALUES(?,?,?,?) ON CONFLICT(provider) DO UPDATE SET
|
||||
next_check_at=excluded.next_check_at,error_message=excluded.error_message,updated_at=excluded.updated_at`))
|
||||
.run(PROVIDER, retryAt.toISOString(), message.slice(0, 240), now.toISOString());
|
||||
}
|
||||
return getRemoveBgCreditStatus();
|
||||
}
|
||||
|
||||
export async function recordRemoveBgCharge(charged: number) {
|
||||
if (!Number.isFinite(charged) || charged <= 0) return;
|
||||
const database = await getDatabase();
|
||||
const row = await (await database.prepare(
|
||||
'SELECT credits_total,free_calls,severity,episode_id FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
if (row?.credits_total === null || row?.credits_total === undefined) return;
|
||||
const now = new Date().toISOString();
|
||||
const total = Math.max(0, Number(row.credits_total) - charged);
|
||||
const nextSeverity = severity(total, Number(row.free_calls ?? 0));
|
||||
const previousSeverity = String(row.severity ?? 'unknown');
|
||||
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
||||
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
||||
const episodeId = startsEpisode ? randomUUID() : String(row.episode_id ?? '') || null;
|
||||
await (await database.prepare(`UPDATE external_service_status SET credits_total=?,last_charge=?,last_charge_at=?,
|
||||
severity=?,episode_id=?,episode_started_at=CASE WHEN ? THEN ? ELSE episode_started_at END,updated_at=?
|
||||
WHERE provider=?`)).run(
|
||||
total, charged, now, nextSeverity, episodeId, startsEpisode ? 1 : 0, now, now, PROVIDER
|
||||
);
|
||||
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
||||
await queueAlertEpisode(episodeId, nextSeverity);
|
||||
}
|
||||
}
|
||||
|
||||
export async function deliverRemoveBgCreditAlerts() {
|
||||
const database = await getDatabase();
|
||||
const pending = await (await database.prepare(`SELECT delivery.*,status.credits_total,status.free_calls
|
||||
FROM external_service_alert_deliveries delivery
|
||||
JOIN external_service_status status ON status.provider=delivery.provider
|
||||
WHERE delivery.delivered_at IS NULL ORDER BY delivery.created_at LIMIT 100`)).all();
|
||||
let delivered = 0;
|
||||
for (const alert of pending) {
|
||||
const title = alert.severity === 'critical'
|
||||
? 'remove.bg credits exhausted'
|
||||
: 'remove.bg credits running low';
|
||||
const body = `remove.bg has ${Number(alert.credits_total ?? 0).toLocaleString()} paid credits and ${Number(alert.free_calls ?? 0).toLocaleString()} free calls remaining. Review image-processing capacity in System Metrics.`;
|
||||
const notificationId = await createNotification({
|
||||
userId: String(alert.user_id),
|
||||
category: 'isac',
|
||||
source: 'ISAC SYSTEM OVERSIGHT',
|
||||
title,
|
||||
body,
|
||||
link: '/isac/metrics',
|
||||
actions: [{ label: 'VIEW SYSTEM METRICS', href: '/isac/metrics' }],
|
||||
mandatory: true
|
||||
});
|
||||
await (await database.prepare(`UPDATE external_service_alert_deliveries
|
||||
SET notification_id=?,delivered_at=? WHERE id=? AND delivered_at IS NULL`))
|
||||
.run(notificationId, new Date().toISOString(), alert.id);
|
||||
delivered++;
|
||||
}
|
||||
return delivered;
|
||||
}
|
||||
|
||||
export async function getRemoveBgCreditStatus(): Promise<RemoveBgCreditStatus> {
|
||||
const database = await getDatabase();
|
||||
const row = await (await database.prepare('SELECT * FROM external_service_status WHERE provider=?')).get(PROVIDER);
|
||||
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
||||
return {
|
||||
configured: Boolean(env.REMOVE_BG_API_KEY?.trim()),
|
||||
total: finiteNonNegative(row?.credits_total),
|
||||
subscription: finiteNonNegative(row?.credits_subscription),
|
||||
payg: finiteNonNegative(row?.credits_payg),
|
||||
enterprise: finiteNonNegative(row?.credits_enterprise),
|
||||
freeCalls: finiteNonNegative(row?.free_calls),
|
||||
checkedAt: row?.checked_at ? String(row.checked_at) : null,
|
||||
nextCheckAt: row?.next_check_at ? String(row.next_check_at) : null,
|
||||
lastCharge: finiteNonNegative(row?.last_charge),
|
||||
lastChargeAt: row?.last_charge_at ? String(row.last_charge_at) : null,
|
||||
error: row?.error_message ? String(row.error_message) : null,
|
||||
severity: (['ok', 'warning', 'critical'].includes(String(row?.severity))
|
||||
? String(row?.severity)
|
||||
: 'unknown') as RemoveBgCreditStatus['severity'],
|
||||
warningCreditThreshold,
|
||||
warningFreeCallThreshold
|
||||
};
|
||||
}
|
||||
|
||||
export async function runRemoveBgCreditMonitor() {
|
||||
await refreshRemoveBgCredits(false);
|
||||
return deliverRemoveBgCreditAlerts();
|
||||
}
|
||||
Reference in New Issue
Block a user