feat: add automated community moderation
This commit is contained in:
JB
@@ -3,13 +3,19 @@ import { getSessionUser } from '$lib/server/auth/session';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import { normalizeUserMarkdown, renderUserMarkdown } from '$lib/markdown';
|
||||
import { createNotification } from '$lib/server/notifications';
|
||||
import {
|
||||
getCommunityModerationPolicy,
|
||||
moderateCommunityText,
|
||||
createUserContentReport,
|
||||
recordCommunityModerationScan
|
||||
} from '$lib/server/community-moderation';
|
||||
import { error, fail } from '@sveltejs/kit';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { queueCurrentAvatarReview } from '$lib/server/avatars';
|
||||
import type { Actions, PageServerLoad } from './$types';
|
||||
|
||||
const COMMENT_MAX_CHARACTERS = 2_000;
|
||||
const RAPID_COMMENT_SECONDS = 10;
|
||||
const DUPLICATE_COMMENT_SECONDS = 300;
|
||||
|
||||
const unavailable = (row: Record<string, unknown> | null | undefined) => {
|
||||
const banned = typeof row?.banned_until === 'string' && new Date(row.banned_until) > new Date();
|
||||
@@ -163,11 +169,13 @@ export const load: PageServerLoad = async ({ params, cookies }) => {
|
||||
comment.avatar_updated_at
|
||||
)
|
||||
},
|
||||
canDelete: viewer?.id === String(comment.author_id)
|
||||
canDelete: viewer?.id === String(comment.author_id),
|
||||
canReport: Boolean(viewer) && viewer?.id !== String(comment.author_id)
|
||||
})),
|
||||
viewer: {
|
||||
id: viewer?.id ?? null,
|
||||
canComment: Boolean(viewer) && Boolean(agent.is_public),
|
||||
canReportProfile: Boolean(viewer) && !isSelf,
|
||||
isSelf
|
||||
}
|
||||
};
|
||||
@@ -187,6 +195,7 @@ export const actions: Actions = {
|
||||
});
|
||||
}
|
||||
const database = await getDatabase();
|
||||
const moderationPolicy = await getCommunityModerationPolicy();
|
||||
const now = new Date();
|
||||
const rows = await (await database.prepare(`SELECT users.id,
|
||||
COALESCE(profile.display_name,users.username) AS display_name,profile.is_public,
|
||||
@@ -207,7 +216,7 @@ export const actions: Actions = {
|
||||
|
||||
const recent = await (await database.prepare(`SELECT comments.body_markdown,comments.created_at
|
||||
FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id
|
||||
WHERE comments.author_id=? AND comments.status='visible'
|
||||
WHERE comments.author_id=? AND comments.status IN ('visible','pending_review')
|
||||
AND thread.target_type='profile' AND thread.target_id=?
|
||||
ORDER BY comments.created_at DESC LIMIT 1`)).get(commenter.id, params.id);
|
||||
if (typeof recent?.created_at === 'string') {
|
||||
@@ -215,11 +224,17 @@ export const actions: Actions = {
|
||||
if (elapsed < RAPID_COMMENT_SECONDS) {
|
||||
return fail(429, { profileCommentError: 'Wait a few seconds before posting again.' });
|
||||
}
|
||||
if (elapsed < DUPLICATE_COMMENT_SECONDS && recent.body_markdown === body) {
|
||||
if (elapsed < moderationPolicy.duplicateWindowSeconds && recent.body_markdown === body) {
|
||||
return fail(409, { profileCommentError: 'That comment was already posted.' });
|
||||
}
|
||||
}
|
||||
|
||||
const moderation = await moderateCommunityText({
|
||||
contentType: 'comment',
|
||||
text: body,
|
||||
policy: moderationPolicy
|
||||
});
|
||||
const commentStatus = moderation.decision === 'approved' ? 'visible' : 'pending_review';
|
||||
const createdAt = now.toISOString();
|
||||
const proposedThreadId = randomUUID();
|
||||
await (await database.prepare(`INSERT OR IGNORE INTO comment_threads
|
||||
@@ -232,9 +247,15 @@ export const actions: Actions = {
|
||||
const commentId = randomUUID();
|
||||
await (await database.prepare(`INSERT INTO comments
|
||||
(id,thread_id,author_id,parent_id,body_markdown,status,created_at,updated_at)
|
||||
VALUES(?,?,?,NULL,?,'visible',?,?)`))
|
||||
.run(commentId, thread.id, commenter.id, body, createdAt, createdAt);
|
||||
if (commenter.id !== params.id) {
|
||||
VALUES(?,?,?,NULL,?,?,?,?)`))
|
||||
.run(commentId, thread.id, commenter.id, body, commentStatus, createdAt, createdAt);
|
||||
await recordCommunityModerationScan({
|
||||
contentType: 'comment',
|
||||
contentId: commentId,
|
||||
authorId: commenter.id,
|
||||
result: moderation
|
||||
});
|
||||
if (commentStatus === 'visible' && commenter.id !== params.id) {
|
||||
await createNotification({
|
||||
userId: params.id,
|
||||
category: 'comments',
|
||||
@@ -250,6 +271,7 @@ export const actions: Actions = {
|
||||
}
|
||||
return {
|
||||
profileCommentSaved: true,
|
||||
profileCommentReviewQueued: commentStatus === 'pending_review',
|
||||
profileCommentId: commentId,
|
||||
profileCommentResultToken: randomUUID()
|
||||
};
|
||||
@@ -278,5 +300,72 @@ export const actions: Actions = {
|
||||
profileCommentId: commentId,
|
||||
profileCommentResultToken: randomUUID()
|
||||
};
|
||||
},
|
||||
|
||||
reportContent: async ({ params, cookies, request }) => {
|
||||
const reporter = await getSessionUser(cookies);
|
||||
const form = await request.formData();
|
||||
const contentType = String(form.get('contentType') ?? '');
|
||||
const contentId = String(form.get('contentId') ?? '').trim();
|
||||
const category = String(form.get('category') ?? '');
|
||||
const details = String(form.get('details') ?? '').normalize('NFKC').trim();
|
||||
const failure = (status: number, message: string) => fail(status, {
|
||||
userReportError: message,
|
||||
userReportContentId: contentId,
|
||||
userReportContentType: contentType,
|
||||
userReportResultToken: randomUUID()
|
||||
});
|
||||
if (!reporter) return failure(401, 'Sign in to request an Oversight review.');
|
||||
if (!['comment', 'profile_bio', 'avatar'].includes(contentType)) {
|
||||
return failure(400, 'Choose reportable content.');
|
||||
}
|
||||
if (!['spam', 'harassment', 'hate', 'sexual', 'violence', 'self_harm', 'privacy', 'other'].includes(category)) {
|
||||
return failure(400, 'Choose the policy concern that best fits this report.');
|
||||
}
|
||||
if (details.length > 500 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(details)) {
|
||||
return failure(400, 'Report details may contain up to 500 visible characters.');
|
||||
}
|
||||
const database = await getDatabase();
|
||||
if (contentType === 'comment') {
|
||||
const comment = await (await database.prepare(`SELECT comments.author_id
|
||||
FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id
|
||||
WHERE comments.id=? AND comments.status='visible'
|
||||
AND thread.target_type='profile' AND thread.target_id=?`)).get(contentId, params.id);
|
||||
if (!comment) return failure(404, 'That comment is no longer available.');
|
||||
if (String(comment.author_id) === reporter.id) return failure(400, 'You cannot report your own comment.');
|
||||
} else {
|
||||
if (contentId !== params.id) return failure(400, 'That profile content does not match this Agent.');
|
||||
if (reporter.id === params.id) return failure(400, 'You cannot report your own profile content.');
|
||||
const profile = await (await database.prepare(`SELECT profile.bio_markdown,profile.is_public,
|
||||
COALESCE(moderation.bio_hidden,0) AS bio_hidden,avatar.source AS avatar_source
|
||||
FROM user_profiles profile
|
||||
LEFT JOIN profile_moderation moderation ON moderation.user_id=profile.user_id
|
||||
LEFT JOIN user_avatars avatar ON avatar.user_id=profile.user_id
|
||||
WHERE profile.user_id=?`)).get(params.id);
|
||||
if (!profile?.is_public) return failure(404, 'That profile is no longer available.');
|
||||
if (contentType === 'profile_bio' && (profile.bio_hidden || !String(profile.bio_markdown ?? '').trim())) {
|
||||
return failure(404, 'That biography is no longer available.');
|
||||
}
|
||||
if (contentType === 'avatar' && profile.avatar_source !== 'upload') {
|
||||
return failure(400, 'Preset SHD avatars are managed site assets and do not need an Agent report.');
|
||||
}
|
||||
}
|
||||
const report = await createUserContentReport({
|
||||
contentType: contentType as 'comment' | 'profile_bio' | 'avatar',
|
||||
contentId,
|
||||
reporterId: reporter.id,
|
||||
category: category as 'spam' | 'harassment' | 'hate' | 'sexual' | 'violence' | 'self_harm' | 'privacy' | 'other',
|
||||
details
|
||||
});
|
||||
if (!report) return failure(409, 'You already have a pending report for this content.');
|
||||
if (contentType === 'avatar') {
|
||||
await queueCurrentAvatarReview(database, params.id, { origin: 'user_report' });
|
||||
}
|
||||
return {
|
||||
userReportSaved: true,
|
||||
userReportContentId: contentId,
|
||||
userReportContentType: contentType,
|
||||
userReportResultToken: randomUUID()
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
<script lang="ts">
|
||||
import AgentIdentityLink from '$lib/components/AgentIdentityLink.svelte';
|
||||
import ContentReport from '$lib/components/ContentReport.svelte';
|
||||
import MarkdownContent from '$lib/components/MarkdownContent.svelte';
|
||||
import MarkdownEditor from '$lib/components/MarkdownEditor.svelte';
|
||||
import ModerationDisclosure from '$lib/components/ModerationDisclosure.svelte';
|
||||
import PageBreadcrumb from '$lib/components/PageBreadcrumb.svelte';
|
||||
import PendingForm from '$lib/components/PendingForm.svelte';
|
||||
|
||||
@@ -32,6 +34,7 @@
|
||||
createdAt: string;
|
||||
author: { id: string; displayName: string; avatarUrl: string };
|
||||
canDelete: boolean;
|
||||
canReport: boolean;
|
||||
};
|
||||
type PageData = {
|
||||
agent: PublicAgent;
|
||||
@@ -40,7 +43,7 @@
|
||||
guides: PublicContent[];
|
||||
authoredComments: AuthoredComment[];
|
||||
profileComments: ProfileComment[];
|
||||
viewer: { id: string | null; canComment: boolean; isSelf: boolean };
|
||||
viewer: { id: string | null; canComment: boolean; canReportProfile: boolean; isSelf: boolean };
|
||||
};
|
||||
|
||||
let { data, form }: { data: PageData; form?: Record<string, any> | null } = $props();
|
||||
@@ -75,6 +78,7 @@
|
||||
<header class="agent-command">
|
||||
<AgentIdentityLink agent={data.agent} size="large" meta="SHD.BUILD AGENT PROFILE" />
|
||||
<div class="agent-command-actions">
|
||||
{#if data.viewer.canReportProfile}<ContentReport contentType="avatar" contentId={data.agent.id} label="REPORT AVATAR" result={form}/>{/if}
|
||||
{#if data.viewer.isSelf}<a class="secondary" href="/account">EDIT MY PROFILE →</a>{/if}
|
||||
</div>
|
||||
</header>
|
||||
@@ -89,7 +93,7 @@
|
||||
<div class="agent-profile-grid">
|
||||
<div class="agent-primary">
|
||||
<section class="profile-section agent-bio">
|
||||
<header><div><p class="eyebrow">FIELD DOSSIER</p><h1>ABOUT {data.agent.displayName.toUpperCase()}</h1></div><span>PUBLIC BIO</span></header>
|
||||
<header><div><p class="eyebrow">FIELD DOSSIER</p><h1>ABOUT {data.agent.displayName.toUpperCase()}</h1></div><div class="profile-header-actions"><span>PUBLIC BIO</span>{#if data.viewer.canReportProfile && data.agent.bioHtml}<ContentReport contentType="profile_bio" contentId={data.agent.id} label="REPORT BIO" result={form}/>{/if}</div></header>
|
||||
<div class="bio-content">
|
||||
<MarkdownContent html={data.agent.bioHtml} emptyMessage="This Agent has not published a field bio yet." />
|
||||
</div>
|
||||
@@ -167,8 +171,11 @@
|
||||
required
|
||||
bind:value={profileCommentDraft}
|
||||
/>
|
||||
<ModerationDisclosure/>
|
||||
{#if form?.profileCommentError || form?.commentError}<p class="error">{form?.profileCommentError ?? form?.commentError}</p>{/if}
|
||||
{#if form?.profileCommentSaved || form?.commentSaved}<p class="success">Profile message transmitted.</p>{/if}
|
||||
{#if form?.profileCommentSaved || form?.commentSaved}
|
||||
<p class="success">{form?.profileCommentReviewQueued ? 'Profile message received and held for Oversight review.' : 'Profile message transmitted.'}</p>
|
||||
{/if}
|
||||
<button class="primary" disabled={!profileCommentDraft.trim()}>POST MESSAGE →</button>
|
||||
</PendingForm>
|
||||
{:else if !data.viewer.id}
|
||||
@@ -185,6 +192,7 @@
|
||||
{#if comment.canDelete}
|
||||
<button type="button" class="comment-remove" aria-expanded={deletingCommentId === comment.id} onclick={() => deletingCommentId = comment.id}>REMOVE</button>
|
||||
{/if}
|
||||
{#if comment.canReport}<ContentReport contentType="comment" contentId={comment.id} result={form}/>{/if}
|
||||
</header>
|
||||
<MarkdownContent html={comment.bodyHtml} compact />
|
||||
{#if deletingCommentId === comment.id}
|
||||
@@ -223,6 +231,7 @@
|
||||
.public-empty{grid-column:1/-1;min-height:150px;display:grid;place-content:center;gap:6px;padding:20px;text-align:center}.public-empty span{color:var(--shd-orange);font-size:28px}.public-empty strong{font-size:var(--shd-type-telemetry);letter-spacing:1px}.public-empty p{margin:0;color:var(--shd-text-muted);font-family:var(--shd-font-editorial);font-size:var(--shd-type-supporting)}.public-empty.compact{min-height:110px}
|
||||
.authored-comment-list{display:grid;gap:7px;padding:7px}.authored-comment-list>article{display:grid;gap:9px;padding:10px;border:1px solid var(--shd-border);background:#0d1011}.authored-comment-list article>header{display:flex;justify-content:space-between;gap:10px}.authored-comment-list article>header>div{display:grid;gap:3px}.authored-comment-list article header span,.authored-comment-list article header small,.authored-comment-list article>a{font-size:var(--shd-type-telemetry);font-weight:900;letter-spacing:.7px}.authored-comment-list article header span,.authored-comment-list article>a{color:var(--shd-orange)}.authored-comment-list article header small{color:var(--shd-text-muted)}.authored-comment-list article>a{justify-self:end}
|
||||
.profile-comms{align-self:start;position:sticky;top:88px}:global(form.profile-comment-form){display:grid;gap:8px;padding:8px;border-bottom:1px solid var(--shd-border)}:global(form.profile-comment-form > button){min-height:40px}.profile-comment-list{display:grid}.profile-comment-list>article{display:grid;gap:9px;padding:10px;border-bottom:1px solid var(--shd-border)}.profile-comment-list>article:last-child{border-bottom:0}.profile-comment-list article>header{display:flex;align-items:center;justify-content:space-between;gap:8px}.profile-comment-list article>header :global(.agent-identity){flex:1}.comment-remove{min-height:32px;padding:0 7px;border:1px solid var(--shd-danger);background:transparent;color:var(--shd-danger);font-size:var(--shd-type-telemetry);font-weight:900;cursor:pointer}.comment-remove:hover,.comment-remove:focus-visible{background:var(--shd-danger);color:var(--shd-bg)}.comment-delete-confirm{display:grid;gap:8px;padding:8px;border:1px solid var(--shd-danger);background:#1b1111}.comment-delete-confirm p{margin:0;color:var(--shd-text-muted);font-size:var(--shd-type-supporting)}.comment-delete-confirm>div{display:flex;justify-content:flex-end;gap:7px}.comment-delete-confirm button{min-height:38px;padding:0 10px}.comment-access-note{display:grid;gap:7px;padding:12px;border-bottom:1px solid var(--shd-border)}.comment-access-note strong{color:var(--shd-orange);font-size:var(--shd-type-telemetry);letter-spacing:.8px}.comment-access-note p{margin:0;color:var(--shd-text-muted);font-size:var(--shd-type-supporting)}.comment-access-note a{justify-self:start}
|
||||
.profile-header-actions{display:flex;align-items:center;gap:7px}.profile-header-actions>span{color:var(--shd-text-muted);font-size:var(--shd-type-telemetry);font-weight:900;letter-spacing:.7px}
|
||||
@media(max-width:1000px){.agent-profile-grid{grid-template-columns:1fr}.profile-comms{position:static}.agent-telemetry{grid-template-columns:repeat(2,minmax(0,1fr))}.agent-telemetry>div:nth-child(3){border-left:0}.agent-telemetry>div:nth-child(n+3){border-top:1px solid var(--shd-border)}}
|
||||
@media(max-width:650px){.agent-command{align-items:stretch;flex-direction:column}.agent-command-actions a{width:100%;min-height:44px;justify-content:center}.agent-telemetry{grid-template-columns:1fr}.agent-telemetry>div{border-top:1px solid var(--shd-border);border-left:0}.agent-telemetry>div:first-child{border-top:0}.public-content-grid{grid-template-columns:1fr}.profile-section>header{align-items:flex-start;flex-direction:column}:global(form.profile-comment-form > button),.comment-delete-confirm button{min-height:44px}.comment-delete-confirm>div{display:grid;grid-template-columns:1fr 1fr}}
|
||||
@media(prefers-reduced-motion:reduce){.public-content-card,.comment-remove{transition:none}}
|
||||
|
||||
Reference in New Issue
Block a user