237 lines
11 KiB
TypeScript
237 lines
11 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { env } from '$env/dynamic/private';
|
|
import { getDatabase } from '$lib/server/database';
|
|
import { createNotification } from '$lib/server/notifications';
|
|
|
|
const PROVIDER = 'remove_bg';
|
|
const ACCOUNT_ENDPOINT = 'https://api.remove.bg/v1.0/account';
|
|
|
|
export type RemoveBgCreditStatus = {
|
|
configured: boolean;
|
|
total: number | null;
|
|
subscription: number | null;
|
|
payg: number | null;
|
|
enterprise: number | null;
|
|
freeCalls: number | null;
|
|
checkedAt: string | null;
|
|
nextCheckAt: string | null;
|
|
lastCharge: number | null;
|
|
lastChargeAt: string | null;
|
|
error: string | null;
|
|
severity: 'unknown' | 'ok' | 'warning' | 'critical';
|
|
warningCreditThreshold: number;
|
|
warningFreeCallThreshold: number;
|
|
};
|
|
|
|
const finiteNonNegative = (value: unknown) => {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
|
};
|
|
|
|
export function parseRemoveBgAccount(payload: unknown) {
|
|
const attributes = (payload as { data?: { attributes?: Record<string, unknown> } })?.data?.attributes;
|
|
const credits = attributes?.credits as Record<string, unknown> | undefined;
|
|
const api = attributes?.api as Record<string, unknown> | undefined;
|
|
const total = finiteNonNegative(credits?.total);
|
|
const freeCalls = finiteNonNegative(api?.free_calls);
|
|
if (total === null || freeCalls === null) throw new Error('remove.bg returned an invalid account balance.');
|
|
return {
|
|
total,
|
|
subscription: finiteNonNegative(credits?.subscription) ?? 0,
|
|
payg: finiteNonNegative(credits?.payg) ?? 0,
|
|
enterprise: finiteNonNegative(credits?.enterprise) ?? 0,
|
|
freeCalls: Math.floor(freeCalls)
|
|
};
|
|
}
|
|
|
|
function settings() {
|
|
const warningCreditThreshold = finiteNonNegative(env.REMOVE_BG_LOW_CREDIT_THRESHOLD) ?? 5;
|
|
const warningFreeCallThreshold = finiteNonNegative(env.REMOVE_BG_LOW_FREE_CALL_THRESHOLD) ?? 10;
|
|
const checkHours = Math.max(1, finiteNonNegative(env.REMOVE_BG_CREDIT_CHECK_HOURS) ?? 6);
|
|
return { warningCreditThreshold, warningFreeCallThreshold, checkHours };
|
|
}
|
|
|
|
function severity(total: number, freeCalls: number) {
|
|
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
|
if (total <= 0 && freeCalls <= 0) return 'critical' as const;
|
|
if (total <= warningCreditThreshold && freeCalls <= warningFreeCallThreshold) return 'warning' as const;
|
|
return 'ok' as const;
|
|
}
|
|
|
|
async function activeAdministrators() {
|
|
const database = await getDatabase();
|
|
return (await database.prepare(`
|
|
SELECT DISTINCT users.id
|
|
FROM users
|
|
LEFT JOIN deleted_accounts deleted ON deleted.user_id=users.id
|
|
LEFT JOIN user_status status ON status.user_id=users.id
|
|
LEFT JOIN user_roles ON user_roles.user_id=users.id
|
|
LEFT JOIN role_permissions ON role_permissions.role_id=user_roles.role_id
|
|
LEFT JOIN permissions role_permission ON role_permission.id=role_permissions.permission_id
|
|
LEFT JOIN user_permissions ON user_permissions.user_id=users.id
|
|
LEFT JOIN permissions direct_permission ON direct_permission.id=user_permissions.permission_id
|
|
WHERE deleted.user_id IS NULL AND status.blocked_at IS NULL
|
|
AND (status.banned_until IS NULL OR status.banned_until<=?)
|
|
AND (users.role='admin' OR user_roles.role_id='role-admin'
|
|
OR role_permission.name='system.admin' OR direct_permission.name='system.admin')
|
|
`)).all(new Date().toISOString());
|
|
}
|
|
|
|
async function queueAlertEpisode(episodeId: string, alertSeverity: 'warning' | 'critical') {
|
|
const database = await getDatabase();
|
|
const now = new Date().toISOString();
|
|
for (const administrator of await activeAdministrators()) {
|
|
await (await database.prepare(`INSERT OR IGNORE INTO external_service_alert_deliveries
|
|
(id,provider,episode_id,severity,user_id,created_at)
|
|
VALUES(?,?,?,?,?,?)`)).run(
|
|
randomUUID(), PROVIDER, episodeId, alertSeverity, administrator.id, now
|
|
);
|
|
}
|
|
}
|
|
|
|
async function persistBalance(balance: ReturnType<typeof parseRemoveBgAccount>) {
|
|
const database = await getDatabase();
|
|
const previous = await (await database.prepare(
|
|
'SELECT severity,episode_id FROM external_service_status WHERE provider=?'
|
|
)).get(PROVIDER);
|
|
const now = new Date();
|
|
const nextCheck = new Date(now.getTime() + settings().checkHours * 60 * 60_000);
|
|
const nextSeverity = severity(balance.total, balance.freeCalls);
|
|
const previousSeverity = String(previous?.severity ?? 'unknown');
|
|
const recovered = nextSeverity === 'ok' && (previousSeverity === 'warning' || previousSeverity === 'critical');
|
|
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
|
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
|
const episodeId = startsEpisode ? randomUUID() : String(previous?.episode_id ?? '') || null;
|
|
await (await database.prepare(`INSERT INTO external_service_status(
|
|
provider,credits_total,credits_subscription,credits_payg,credits_enterprise,free_calls,
|
|
checked_at,next_check_at,error_message,severity,episode_id,episode_started_at,recovered_at,updated_at
|
|
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(provider) DO UPDATE SET
|
|
credits_total=excluded.credits_total,credits_subscription=excluded.credits_subscription,
|
|
credits_payg=excluded.credits_payg,credits_enterprise=excluded.credits_enterprise,
|
|
free_calls=excluded.free_calls,checked_at=excluded.checked_at,next_check_at=excluded.next_check_at,
|
|
error_message=NULL,severity=excluded.severity,episode_id=excluded.episode_id,
|
|
episode_started_at=CASE WHEN excluded.episode_id<>external_service_status.episode_id THEN excluded.episode_started_at ELSE external_service_status.episode_started_at END,
|
|
recovered_at=excluded.recovered_at,updated_at=excluded.updated_at`)).run(
|
|
PROVIDER, balance.total, balance.subscription, balance.payg, balance.enterprise, balance.freeCalls,
|
|
now.toISOString(), nextCheck.toISOString(), null, nextSeverity,
|
|
recovered ? null : episodeId, startsEpisode ? now.toISOString() : null,
|
|
recovered ? now.toISOString() : null, now.toISOString()
|
|
);
|
|
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
|
await queueAlertEpisode(episodeId, nextSeverity);
|
|
}
|
|
}
|
|
|
|
export async function refreshRemoveBgCredits(force = false) {
|
|
const database = await getDatabase();
|
|
const existing = await (await database.prepare(
|
|
'SELECT next_check_at FROM external_service_status WHERE provider=?'
|
|
)).get(PROVIDER);
|
|
if (!force && existing?.next_check_at && String(existing.next_check_at) > new Date().toISOString()) {
|
|
return getRemoveBgCreditStatus();
|
|
}
|
|
const apiKey = env.REMOVE_BG_API_KEY?.trim();
|
|
if (!apiKey) return getRemoveBgCreditStatus();
|
|
try {
|
|
const response = await fetch(ACCOUNT_ENDPOINT, {
|
|
headers: { 'X-Api-Key': apiKey },
|
|
signal: AbortSignal.timeout(15_000)
|
|
});
|
|
if (!response.ok) throw new Error(`remove.bg account check failed with status ${response.status}.`);
|
|
await persistBalance(parseRemoveBgAccount(await response.json()));
|
|
} catch (error) {
|
|
const now = new Date();
|
|
const retryAt = new Date(now.getTime() + 60 * 60_000);
|
|
const message = error instanceof Error ? error.message : 'remove.bg account check failed.';
|
|
await (await database.prepare(`INSERT INTO external_service_status(provider,next_check_at,error_message,updated_at)
|
|
VALUES(?,?,?,?) ON CONFLICT(provider) DO UPDATE SET
|
|
next_check_at=excluded.next_check_at,error_message=excluded.error_message,updated_at=excluded.updated_at`))
|
|
.run(PROVIDER, retryAt.toISOString(), message.slice(0, 240), now.toISOString());
|
|
}
|
|
return getRemoveBgCreditStatus();
|
|
}
|
|
|
|
export async function recordRemoveBgCharge(charged: number) {
|
|
if (!Number.isFinite(charged) || charged <= 0) return;
|
|
const database = await getDatabase();
|
|
const row = await (await database.prepare(
|
|
'SELECT credits_total,free_calls,severity,episode_id FROM external_service_status WHERE provider=?'
|
|
)).get(PROVIDER);
|
|
if (row?.credits_total === null || row?.credits_total === undefined) return;
|
|
const now = new Date().toISOString();
|
|
const total = Math.max(0, Number(row.credits_total) - charged);
|
|
const nextSeverity = severity(total, Number(row.free_calls ?? 0));
|
|
const previousSeverity = String(row.severity ?? 'unknown');
|
|
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
|
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
|
const episodeId = startsEpisode ? randomUUID() : String(row.episode_id ?? '') || null;
|
|
await (await database.prepare(`UPDATE external_service_status SET credits_total=?,last_charge=?,last_charge_at=?,
|
|
severity=?,episode_id=?,episode_started_at=CASE WHEN ? THEN ? ELSE episode_started_at END,updated_at=?
|
|
WHERE provider=?`)).run(
|
|
total, charged, now, nextSeverity, episodeId, startsEpisode ? 1 : 0, now, now, PROVIDER
|
|
);
|
|
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
|
await queueAlertEpisode(episodeId, nextSeverity);
|
|
}
|
|
}
|
|
|
|
export async function deliverRemoveBgCreditAlerts() {
|
|
const database = await getDatabase();
|
|
const pending = await (await database.prepare(`SELECT delivery.*,status.credits_total,status.free_calls
|
|
FROM external_service_alert_deliveries delivery
|
|
JOIN external_service_status status ON status.provider=delivery.provider
|
|
WHERE delivery.delivered_at IS NULL ORDER BY delivery.created_at LIMIT 100`)).all();
|
|
let delivered = 0;
|
|
for (const alert of pending) {
|
|
const title = alert.severity === 'critical'
|
|
? 'remove.bg credits exhausted'
|
|
: 'remove.bg credits running low';
|
|
const body = `remove.bg has ${Number(alert.credits_total ?? 0).toLocaleString()} paid credits and ${Number(alert.free_calls ?? 0).toLocaleString()} free calls remaining. Review image-processing capacity in System Metrics.`;
|
|
const notificationId = await createNotification({
|
|
userId: String(alert.user_id),
|
|
category: 'isac',
|
|
source: 'ISAC SYSTEM OVERSIGHT',
|
|
title,
|
|
body,
|
|
link: '/isac/metrics',
|
|
actions: [{ label: 'VIEW SYSTEM METRICS', href: '/isac/metrics' }],
|
|
mandatory: true
|
|
});
|
|
await (await database.prepare(`UPDATE external_service_alert_deliveries
|
|
SET notification_id=?,delivered_at=? WHERE id=? AND delivered_at IS NULL`))
|
|
.run(notificationId, new Date().toISOString(), alert.id);
|
|
delivered++;
|
|
}
|
|
return delivered;
|
|
}
|
|
|
|
export async function getRemoveBgCreditStatus(): Promise<RemoveBgCreditStatus> {
|
|
const database = await getDatabase();
|
|
const row = await (await database.prepare('SELECT * FROM external_service_status WHERE provider=?')).get(PROVIDER);
|
|
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
|
return {
|
|
configured: Boolean(env.REMOVE_BG_API_KEY?.trim()),
|
|
total: finiteNonNegative(row?.credits_total),
|
|
subscription: finiteNonNegative(row?.credits_subscription),
|
|
payg: finiteNonNegative(row?.credits_payg),
|
|
enterprise: finiteNonNegative(row?.credits_enterprise),
|
|
freeCalls: finiteNonNegative(row?.free_calls),
|
|
checkedAt: row?.checked_at ? String(row.checked_at) : null,
|
|
nextCheckAt: row?.next_check_at ? String(row.next_check_at) : null,
|
|
lastCharge: finiteNonNegative(row?.last_charge),
|
|
lastChargeAt: row?.last_charge_at ? String(row.last_charge_at) : null,
|
|
error: row?.error_message ? String(row.error_message) : null,
|
|
severity: (['ok', 'warning', 'critical'].includes(String(row?.severity))
|
|
? String(row?.severity)
|
|
: 'unknown') as RemoveBgCreditStatus['severity'],
|
|
warningCreditThreshold,
|
|
warningFreeCallThreshold
|
|
};
|
|
}
|
|
|
|
export async function runRemoveBgCreditMonitor() {
|
|
await refreshRemoveBgCredits(false);
|
|
return deliverRemoveBgCreditAlerts();
|
|
}
|