jb/shd.build
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add automated community moderation

Browse Source
This commit is contained in:
JB
2026-07-27 16:09:56 -04:00
parent 6b76110b76
commit 27cbeca0ed
24 changed files with 3241 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/agents/[id]/+page.server.ts
+96 -7
View File
@@ -3,13 +3,19 @@ import { getSessionUser } from '$lib/server/auth/session';
import { getDatabase } from '$lib/server/database';
import { normalizeUserMarkdown, renderUserMarkdown } from '$lib/markdown';
import { createNotification } from '$lib/server/notifications';
import {
getCommunityModerationPolicy,
moderateCommunityText,
createUserContentReport,
recordCommunityModerationScan
} from '$lib/server/community-moderation';
import { error, fail } from '@sveltejs/kit';
import { randomUUID } from 'node:crypto';
import { queueCurrentAvatarReview } from '$lib/server/avatars';
import type { Actions, PageServerLoad } from './$types';
const COMMENT_MAX_CHARACTERS = 2_000;
const RAPID_COMMENT_SECONDS = 10;
const DUPLICATE_COMMENT_SECONDS = 300;
const unavailable = (row: Record<string, unknown> | null | undefined) => {
const banned = typeof row?.banned_until === 'string' && new Date(row.banned_until) > new Date();
@@ -163,11 +169,13 @@ export const load: PageServerLoad = async ({ params, cookies }) => {
comment.avatar_updated_at
)
},
canDelete: viewer?.id === String(comment.author_id)
canDelete: viewer?.id === String(comment.author_id),
canReport: Boolean(viewer) && viewer?.id !== String(comment.author_id)
})),
viewer: {
id: viewer?.id ?? null,
canComment: Boolean(viewer) && Boolean(agent.is_public),
canReportProfile: Boolean(viewer) && !isSelf,
isSelf
}
};
@@ -187,6 +195,7 @@ export const actions: Actions = {
});
}
const database = await getDatabase();
const moderationPolicy = await getCommunityModerationPolicy();
const now = new Date();
const rows = await (await database.prepare(`SELECT users.id,
COALESCE(profile.display_name,users.username) AS display_name,profile.is_public,
@@ -207,7 +216,7 @@ export const actions: Actions = {
const recent = await (await database.prepare(`SELECT comments.body_markdown,comments.created_at
FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id
WHERE comments.author_id=? AND comments.status='visible'
WHERE comments.author_id=? AND comments.status IN ('visible','pending_review')
AND thread.target_type='profile' AND thread.target_id=?
ORDER BY comments.created_at DESC LIMIT 1`)).get(commenter.id, params.id);
if (typeof recent?.created_at === 'string') {
@@ -215,11 +224,17 @@ export const actions: Actions = {
if (elapsed < RAPID_COMMENT_SECONDS) {
return fail(429, { profileCommentError: 'Wait a few seconds before posting again.' });
}
if (elapsed < DUPLICATE_COMMENT_SECONDS && recent.body_markdown === body) {
if (elapsed < moderationPolicy.duplicateWindowSeconds && recent.body_markdown === body) {
return fail(409, { profileCommentError: 'That comment was already posted.' });
}
}
const moderation = await moderateCommunityText({
contentType: 'comment',
text: body,
policy: moderationPolicy
});
const commentStatus = moderation.decision === 'approved' ? 'visible' : 'pending_review';
const createdAt = now.toISOString();
const proposedThreadId = randomUUID();
await (await database.prepare(`INSERT OR IGNORE INTO comment_threads
@@ -232,9 +247,15 @@ export const actions: Actions = {
const commentId = randomUUID();
await (await database.prepare(`INSERT INTO comments
(id,thread_id,author_id,parent_id,body_markdown,status,created_at,updated_at)
VALUES(?,?,?,NULL,?,'visible',?,?)`))
.run(commentId, thread.id, commenter.id, body, createdAt, createdAt);
if (commenter.id !== params.id) {
VALUES(?,?,?,NULL,?,?,?,?)`))
.run(commentId, thread.id, commenter.id, body, commentStatus, createdAt, createdAt);
await recordCommunityModerationScan({
contentType: 'comment',
contentId: commentId,
authorId: commenter.id,
result: moderation
});
if (commentStatus === 'visible' && commenter.id !== params.id) {
await createNotification({
userId: params.id,
category: 'comments',
@@ -250,6 +271,7 @@ export const actions: Actions = {
}
return {
profileCommentSaved: true,
profileCommentReviewQueued: commentStatus === 'pending_review',
profileCommentId: commentId,
profileCommentResultToken: randomUUID()
};
@@ -278,5 +300,72 @@ export const actions: Actions = {
profileCommentId: commentId,
profileCommentResultToken: randomUUID()
};
},
reportContent: async ({ params, cookies, request }) => {
const reporter = await getSessionUser(cookies);
const form = await request.formData();
const contentType = String(form.get('contentType') ?? '');
const contentId = String(form.get('contentId') ?? '').trim();
const category = String(form.get('category') ?? '');
const details = String(form.get('details') ?? '').normalize('NFKC').trim();
const failure = (status: number, message: string) => fail(status, {
userReportError: message,
userReportContentId: contentId,
userReportContentType: contentType,
userReportResultToken: randomUUID()
});
if (!reporter) return failure(401, 'Sign in to request an Oversight review.');
if (!['comment', 'profile_bio', 'avatar'].includes(contentType)) {
return failure(400, 'Choose reportable content.');
}
if (!['spam', 'harassment', 'hate', 'sexual', 'violence', 'self_harm', 'privacy', 'other'].includes(category)) {
return failure(400, 'Choose the policy concern that best fits this report.');
}
if (details.length > 500 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(details)) {
return failure(400, 'Report details may contain up to 500 visible characters.');
}
const database = await getDatabase();
if (contentType === 'comment') {
const comment = await (await database.prepare(`SELECT comments.author_id
FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id
WHERE comments.id=? AND comments.status='visible'
AND thread.target_type='profile' AND thread.target_id=?`)).get(contentId, params.id);
if (!comment) return failure(404, 'That comment is no longer available.');
if (String(comment.author_id) === reporter.id) return failure(400, 'You cannot report your own comment.');
} else {
if (contentId !== params.id) return failure(400, 'That profile content does not match this Agent.');
if (reporter.id === params.id) return failure(400, 'You cannot report your own profile content.');
const profile = await (await database.prepare(`SELECT profile.bio_markdown,profile.is_public,
COALESCE(moderation.bio_hidden,0) AS bio_hidden,avatar.source AS avatar_source
FROM user_profiles profile
LEFT JOIN profile_moderation moderation ON moderation.user_id=profile.user_id
LEFT JOIN user_avatars avatar ON avatar.user_id=profile.user_id
WHERE profile.user_id=?`)).get(params.id);
if (!profile?.is_public) return failure(404, 'That profile is no longer available.');
if (contentType === 'profile_bio' && (profile.bio_hidden || !String(profile.bio_markdown ?? '').trim())) {
return failure(404, 'That biography is no longer available.');
}
if (contentType === 'avatar' && profile.avatar_source !== 'upload') {
return failure(400, 'Preset SHD avatars are managed site assets and do not need an Agent report.');
}
}
const report = await createUserContentReport({
contentType: contentType as 'comment' | 'profile_bio' | 'avatar',
contentId,
reporterId: reporter.id,
category: category as 'spam' | 'harassment' | 'hate' | 'sexual' | 'violence' | 'self_harm' | 'privacy' | 'other',
details
});
if (!report) return failure(409, 'You already have a pending report for this content.');
if (contentType === 'avatar') {
await queueCurrentAvatarReview(database, params.id, { origin: 'user_report' });
}
return {
userReportSaved: true,
userReportContentId: contentId,
userReportContentType: contentType,
userReportResultToken: randomUUID()
};
}
};