diff --git a/.env.example b/.env.example index 9e3c776..1ca1668 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ OPENAI_MECHANIC_API_KEY= OPENAI_MECHANIC_MODEL=gpt-5.6-terra OPENAI_MECHANIC_REASONING_EFFORT=medium +OPENAI_MODERATION_API_KEY= # Local path used by the embedded @tursodatabase/database engine. # Relative paths are resolved from the project directory. Development and diff --git a/database/schema.sql b/database/schema.sql index 669d364..301a131 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS avatar_reviews ( storage_key TEXT NOT NULL, byte_size INTEGER NOT NULL, status TEXT NOT NULL CHECK(status IN ('pending', 'approved', 'removed', 'withdrawn', 'superseded')), + review_origin TEXT NOT NULL DEFAULT 'legacy' CHECK(review_origin IN ('automated', 'user_report', 'legacy')), submitted_at TEXT NOT NULL, reviewed_at TEXT, reviewed_by TEXT REFERENCES users(id), @@ -150,7 +151,7 @@ CREATE TABLE IF NOT EXISTS comments ( author_id TEXT NOT NULL REFERENCES users(id), parent_id TEXT REFERENCES comments(id), body_markdown TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'visible' CHECK(status IN ('visible', 'removed')), + status TEXT NOT NULL DEFAULT 'pending_review' CHECK(status IN ('pending_review', 'visible', 'removed')), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, removed_at TEXT, @@ -159,6 +160,170 @@ CREATE TABLE IF NOT EXISTS comments ( ); CREATE INDEX IF NOT EXISTS comments_thread_status_created_idx ON comments(thread_id, status, created_at DESC); CREATE INDEX IF NOT EXISTS comments_author_status_created_idx ON comments(author_id, status, created_at DESC); +CREATE TABLE IF NOT EXISTS community_moderation_settings ( + id TEXT PRIMARY KEY CHECK(id='primary'), + automated_enabled INTEGER NOT NULL DEFAULT 0 CHECK(automated_enabled IN (0,1)), + auto_publish_clean INTEGER NOT NULL DEFAULT 1 CHECK(auto_publish_clean IN (0,1)), + hold_on_provider_error INTEGER NOT NULL DEFAULT 1 CHECK(hold_on_provider_error IN (0,1)), + scan_comments INTEGER NOT NULL DEFAULT 1 CHECK(scan_comments IN (0,1)), + scan_profiles INTEGER NOT NULL DEFAULT 1 CHECK(scan_profiles IN (0,1)), + scan_guides INTEGER NOT NULL DEFAULT 1 CHECK(scan_guides IN (0,1)), + scan_avatars INTEGER NOT NULL DEFAULT 1 CHECK(scan_avatars IN (0,1)), + provider_model TEXT NOT NULL DEFAULT 'omni-moderation-latest' CHECK(length(trim(provider_model))>0), + detect_pii INTEGER NOT NULL DEFAULT 1 CHECK(detect_pii IN (0,1)), + max_links INTEGER NOT NULL DEFAULT 3 CHECK(max_links BETWEEN 0 AND 100), + duplicate_window_seconds INTEGER NOT NULL DEFAULT 300 CHECK(duplicate_window_seconds BETWEEN 0 AND 86400), + policy_version INTEGER NOT NULL DEFAULT 1 CHECK(policy_version>=1), + updated_at TEXT NOT NULL, + updated_by TEXT REFERENCES users(id) +); +INSERT OR IGNORE INTO community_moderation_settings( + id,automated_enabled,auto_publish_clean,hold_on_provider_error, + scan_comments,scan_profiles,scan_guides,scan_avatars,provider_model,detect_pii, + max_links,duplicate_window_seconds,policy_version,updated_at +) VALUES( + 'primary',0,1,1,1,1,1,1,'omni-moderation-latest',1,3,300,1, + '2026-07-27T00:00:00.000Z' +); +CREATE TABLE IF NOT EXISTS community_moderation_thresholds ( + category TEXT PRIMARY KEY CHECK(length(trim(category))>0), + review_threshold REAL NOT NULL CHECK(review_threshold BETWEEN 0 AND 1), + priority_threshold REAL NOT NULL CHECK(priority_threshold BETWEEN 0 AND 1 AND priority_threshold>=review_threshold), + enabled INTEGER NOT NULL DEFAULT 1 CHECK(enabled IN (0,1)), + updated_at TEXT NOT NULL, + updated_by TEXT REFERENCES users(id) +); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('harassment',0.30,0.70,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('harassment/threatening',0.10,0.40,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('hate',0.20,0.60,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('hate/threatening',0.10,0.40,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('illicit',0.30,0.70,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('illicit/violent',0.10,0.40,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('self-harm',0.15,0.50,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('self-harm/intent',0.10,0.35,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('self-harm/instructions',0.10,0.35,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('sexual',0.25,0.65,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('sexual/minors',0.05,0.20,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('violence',0.35,0.75,1,'2026-07-27T00:00:00.000Z'); +INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at) + VALUES('violence/graphic',0.20,0.55,1,'2026-07-27T00:00:00.000Z'); +CREATE TABLE IF NOT EXISTS community_moderation_terms ( + id TEXT PRIMARY KEY, + list_kind TEXT NOT NULL CHECK(list_kind IN ('allow','ban')), + match_mode TEXT NOT NULL CHECK(match_mode IN ('word','phrase')), + value TEXT NOT NULL CHECK(length(trim(value))>0), + normalized_value TEXT NOT NULL CHECK(length(trim(normalized_value))>0), + scopes_json TEXT NOT NULL DEFAULT '["comment","profile_bio","guide"]', + enabled INTEGER NOT NULL DEFAULT 1 CHECK(enabled IN (0,1)), + note TEXT, + created_at TEXT NOT NULL, + created_by TEXT NOT NULL REFERENCES users(id), + updated_at TEXT NOT NULL, + updated_by TEXT NOT NULL REFERENCES users(id), + UNIQUE(list_kind,match_mode,normalized_value,scopes_json) +); +CREATE INDEX IF NOT EXISTS community_moderation_terms_enabled_kind_idx + ON community_moderation_terms(enabled,list_kind,normalized_value); +CREATE TABLE IF NOT EXISTS moderated_content_revisions ( + id TEXT PRIMARY KEY, + content_type TEXT NOT NULL CHECK(content_type IN ('profile_bio','guide')), + content_id TEXT NOT NULL, + submitted_by TEXT NOT NULL REFERENCES users(id), + body_markdown TEXT NOT NULL, + content_hash TEXT NOT NULL CHECK(length(content_hash)>0), + created_at TEXT NOT NULL, + UNIQUE(content_type,content_id,content_hash) +); +CREATE INDEX IF NOT EXISTS moderated_content_revisions_content_created_idx + ON moderated_content_revisions(content_type,content_id,created_at DESC); +CREATE TABLE IF NOT EXISTS content_moderation_scans ( + id TEXT PRIMARY KEY, + content_type TEXT NOT NULL CHECK(content_type IN ('comment','profile_bio','guide','avatar')), + content_id TEXT NOT NULL, + author_id TEXT NOT NULL REFERENCES users(id), + content_hash TEXT NOT NULL CHECK(length(content_hash)>0), + revision_id TEXT REFERENCES moderated_content_revisions(id), + policy_version INTEGER NOT NULL CHECK(policy_version>=1), + provider TEXT NOT NULL CHECK(length(trim(provider))>0), + model TEXT NOT NULL CHECK(length(trim(model))>0), + provider_request_id TEXT, + status TEXT NOT NULL CHECK(status IN ('approved','review','priority_review','error')), + risk_level TEXT NOT NULL CHECK(risk_level IN ('low','medium','high')), + reason TEXT NOT NULL CHECK(length(trim(reason))>0), + scores_json TEXT NOT NULL DEFAULT '{}', + flags_json TEXT NOT NULL DEFAULT '[]', + rule_matches_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + completed_at TEXT, + error_code TEXT, + error_message TEXT, + UNIQUE(content_type,content_id,content_hash,policy_version,provider,model), + CHECK( + (content_type IN ('comment','avatar') AND revision_id IS NULL) + OR (content_type IN ('profile_bio','guide') AND revision_id IS NOT NULL) + ), + CHECK( + (status='error' AND error_code IS NOT NULL) + OR (status<>'error' AND error_code IS NULL AND error_message IS NULL) + ) +); +CREATE INDEX IF NOT EXISTS content_moderation_scans_pending_queue_idx + ON content_moderation_scans(status,created_at) + WHERE status IN ('review','priority_review'); +CREATE INDEX IF NOT EXISTS content_moderation_scans_content_idx + ON content_moderation_scans(content_type,content_id,created_at DESC); +CREATE TABLE IF NOT EXISTS content_moderation_reviews ( + id TEXT PRIMARY KEY, + scan_id TEXT NOT NULL REFERENCES content_moderation_scans(id), + reviewer_id TEXT NOT NULL REFERENCES users(id), + decision TEXT NOT NULL CHECK(decision IN ('approved','rejected','superseded')), + reason TEXT NOT NULL CHECK(length(trim(reason))>0), + created_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS content_moderation_reviews_scan_created_idx + ON content_moderation_reviews(scan_id,created_at DESC); +CREATE INDEX IF NOT EXISTS content_moderation_reviews_reviewer_created_idx + ON content_moderation_reviews(reviewer_id,created_at DESC); +CREATE TABLE IF NOT EXISTS user_content_reports ( + id TEXT PRIMARY KEY, + content_type TEXT NOT NULL CHECK(content_type IN ('comment','profile_bio','guide','avatar')), + content_id TEXT NOT NULL, + reporter_id TEXT NOT NULL REFERENCES users(id), + category TEXT NOT NULL CHECK(category IN ( + 'spam','harassment','hate','sexual','violence','self_harm','privacy','other' + )), + details TEXT, + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','resolved','dismissed')), + created_at TEXT NOT NULL, + reviewed_at TEXT, + reviewed_by TEXT REFERENCES users(id), + resolution_reason TEXT, + CHECK( + (status='pending' AND reviewed_at IS NULL AND reviewed_by IS NULL AND resolution_reason IS NULL) + OR (status IN ('resolved','dismissed') AND reviewed_at IS NOT NULL AND reviewed_by IS NOT NULL + AND resolution_reason IS NOT NULL AND length(trim(resolution_reason))>0) + ) +); +CREATE INDEX IF NOT EXISTS user_content_reports_pending_queue_idx + ON user_content_reports(created_at) + WHERE status='pending'; +CREATE INDEX IF NOT EXISTS user_content_reports_content_idx + ON user_content_reports(content_type,content_id,created_at DESC); +CREATE UNIQUE INDEX IF NOT EXISTS user_content_reports_one_pending_idx + ON user_content_reports(reporter_id,content_type,content_id) + WHERE status='pending'; CREATE TABLE IF NOT EXISTS favorites (user_id TEXT NOT NULL REFERENCES users(id), build_id TEXT NOT NULL REFERENCES builds(id) ON DELETE CASCADE, created_at TEXT NOT NULL, PRIMARY KEY(user_id, build_id)); CREATE TABLE IF NOT EXISTS equipment_types (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, category TEXT NOT NULL); CREATE TABLE IF NOT EXISTS stat_definitions (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, value_kind TEXT NOT NULL, min_value REAL, max_value REAL, unit TEXT, active INTEGER NOT NULL DEFAULT 1); diff --git a/docs/CONTENT_MODERATION.md b/docs/CONTENT_MODERATION.md new file mode 100644 index 0000000..78e3427 --- /dev/null +++ b/docs/CONTENT_MODERATION.md @@ -0,0 +1,124 @@ +# Community content moderation + +SHD.build applies one moderation lifecycle to public comments, profile +biographies, custom avatars, guides, and build descriptions. The shared server +service lives in `src/lib/server/community-moderation.ts`. Content save actions +must use that service before publication. + +## Publication decisions + +The moderation service combines local rules with the configured provider +result: + +- Clean content publishes without human review when automatic publication is + enabled. +- Banned terms, PII checks, link limits, provider flags, and configured score + thresholds send content to Community Oversight. +- Provider failures send content to Oversight when fail-closed behavior is + enabled. +- An allow term can suppress the matching local term rule. It cannot override a + provider flag or threshold. +- A user report sends visible content to Oversight. One report does not hide + the content because reports can be mistaken or abusive. + +The service records every automated scan in `content_moderation_scans`. +Automated scans do not write to the human ISAC audit log. Human policy changes +and review decisions use the shared audit utility. + +## Content storage + +Comments use an explicit status: + +- `visible`: public and approved +- `pending_review`: hidden while Oversight decides +- `removed`: unavailable after author removal or an Oversight decision + +Long-form content uses immutable pending revisions. Profile and guide edits that +need review go into `moderated_content_revisions`; the public row keeps its last +approved text. Approval copies the reviewed revision into the public row. +Rejection preserves the revision and review evidence without publishing it. + +Custom avatar uploads remain active after a clean image scan. A flagged image +or fail-closed scan error creates an `avatar_reviews` item and is unavailable +through the public avatar route until approval. An Agent report also creates a +review item, but leaves an already-published avatar visible while Oversight +decides. + +## ISAC responsibilities + +ISAC Site Controls stores the site-wide policy: + +- provider scanning and per-content enablement; +- clean-content and provider-failure behavior; +- provider model; +- PII, link, and duplicate checks; +- category review and priority thresholds; +- allowed and banned local terms. + +Site Controls requires `site.settings.manage`. Community Oversight handles text +decisions through the existing comment and profile permissions. Avatar +Oversight handles custom images through `avatars.review`. + +Moderators must record a reason when they approve, reject, remove, restore, +dismiss, or resolve a queued item. The action must append an audit event after +the content and review records change. + +## User reports + +Authenticated Agents can report visible content they do not own. The report +control must identify the content type and stable content ID, validate that the +content remains public, and create a pending `user_content_reports` row. +Duplicate pending reports from the same Agent for the same content are rejected. + +A reviewer either dismisses the report and keeps the content or resolves the +report through a moderation action. Report details must not copy authentication +secrets or unnecessary personal information into the audit log. + +## Adding a public UGC field + +Guide authoring, build descriptions, and later public text fields must complete +this checklist: + +1. Normalize and validate the submitted text before scanning. +2. Call `moderateCommunityText` with a supported content type. +3. Place the shared `ModerationDisclosure` beside the submission control. +4. Publish only an `approved` result. +5. Store held long-form text as an immutable revision so the last approved + version remains public. +6. Call `recordCommunityModerationScan` with the stable content ID, author, hash, + policy version, provider evidence, and local-rule matches. +7. Add the content type to the Community Oversight queue and permission checks. +8. Add the shared report control to the public rendering. +9. Delay follower, owner, or reply notifications until the content becomes + public. +10. Add approval, rejection, report dismissal, author notification, and audit + tests. +11. Confirm every public query selects approved content explicitly. + +Do not add a route-specific word filter, provider client, threshold table, or +moderation status vocabulary. Extend the shared service and schema when a new +content type needs more evidence or a different review presentation. + +## Operations and calibration + +The API key stays in `OPENAI_MODERATION_API_KEY`. The server can use +`OPENAI_MECHANIC_API_KEY` as a temporary fallback, but production should use a +separate moderation credential. + +Public submission notices link to the Terms & Privacy page instead of +summarizing provider data handling beside each form. The Terms & Privacy page +must not promise that OpenAI never retains or uses moderation inputs. It states +that OpenAI controls downstream processing under its then-current terms, data +controls, and account settings. If SHD.build enables an OpenAI data-sharing +option, update the legal copy before sending user content under that setting. + +Threshold changes increment the policy version. Stored scans retain their model +and policy version so reviewers can interpret older decisions. Test threshold +changes against representative Division terminology before activation, +including weapon names, damage descriptions, kills, status effects, and build +strategy language. + +Database migrations that add moderation states or revision storage follow the +repository database-preservation procedure: stop this project's development +server, verify the database path, create and verify a JSON backup, migrate, +check foreign keys and critical record counts, then restart and health-check. diff --git a/docs/DESIGN_SYSTEM.md b/docs/DESIGN_SYSTEM.md index 975105e..95d3224 100644 --- a/docs/DESIGN_SYSTEM.md +++ b/docs/DESIGN_SYSTEM.md @@ -468,6 +468,27 @@ Public biographies and community comments use the shared `MarkdownEditor` and remaining-character count, concise help, a readable editorial typeface, and a minimum 44px touch target for toolbar actions on narrow screens. It supports headings, lists, quotes, links, emphasis, and inline code. Raw HTML is escaped. + +Public user-generated content uses the shared `ContentReport` control when the +viewer is authenticated and does not own the content. Keep the control visually +secondary to reading and reply actions. Its disclosure names the concern, +accepts optional bounded context, and states that a report requests human review +without removing content immediately. On narrow screens, present the report +form as a fixed bottom panel with full-width touch controls. + +Submission forms for public user-generated text and custom images use the +shared `ModerationDisclosure` notice. Place it after the input and before +validation or submission controls. The always-visible notice states that +automated safety review may occur and links to the Terms & Privacy section for +the details. Do not hide this disclosure in a hover-only tooltip. + +Automated moderation is exception-based. Clean content publishes without an +Oversight task when policy permits. Flagged comments remain hidden with an +explicit pending state; flagged long-form edits leave the last approved version +public. Flagged custom avatars use the default avatar publicly until approval. +Community Oversight shows automated evidence and Agent reports in one pending +queue, but labels their sources separately. Human approval, rejection, removal, +and report dismissal require a reason and confirmation. Only HTTP and HTTPS links are active; rendered links open separately with UGC, nofollow, and opener-isolation attributes. `MarkdownContent` must receive HTML from the server renderer, never raw user input or client-rendered Markdown. diff --git a/docs/architecture.md b/docs/architecture.md index 93cb4e4..b906569 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,6 +82,24 @@ explains registration closure. This guard runs on the server, so a crafted form submission or OAuth callback cannot create an account while registration is closed. +## Community content moderation + +Comments, profile biographies, custom avatars, guides, and build descriptions +share the moderation lifecycle documented in +[`CONTENT_MODERATION.md`](CONTENT_MODERATION.md). Local rules and the configured +provider classify each submission before publication. Clean submissions can +publish without human work. Flagged submissions and fail-closed provider errors +enter the appropriate Oversight queue. + +Agents may report visible content they do not own. Reports enter Oversight +without hiding the content. Human decisions require a reason and append an +audit event. Long-form edits that need review use immutable pending revisions +so the last approved biography, guide, or description remains public. + +ISAC Site Controls owns provider behavior, content scopes, thresholds, link and +PII checks, and allowed or banned terms. API credentials remain in server +environment configuration. + ## Increment plan 1. Product shell and anonymous local-draft builder. diff --git a/scripts/db-migrate.mjs b/scripts/db-migrate.mjs index 1a13f1a..75e62a7 100644 --- a/scripts/db-migrate.mjs +++ b/scripts/db-migrate.mjs @@ -171,7 +171,7 @@ if (typeof commentsDefinition?.sql === 'string' && commentsDefinition.sql.includ author_id TEXT NOT NULL REFERENCES users(id), parent_id TEXT REFERENCES comments(id), body_markdown TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'visible' CHECK(status IN ('visible', 'removed')), + status TEXT NOT NULL DEFAULT 'pending_review' CHECK(status IN ('pending_review', 'visible', 'removed')), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, removed_at TEXT, @@ -196,6 +196,158 @@ if (typeof commentsDefinition?.sql === 'string' && commentsDefinition.sql.includ } } +const normalizedCommentsDefinition = await (await database.prepare( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='comments'" +)).get(); +if ( + typeof normalizedCommentsDefinition?.sql === 'string' + && !normalizedCommentsDefinition.sql.includes("'pending_review'") +) { + console.log('Adding pre-publication review status to community comments.'); + await database.exec('PRAGMA foreign_keys=OFF'); + await database.exec('BEGIN'); + try { + await database.exec('ALTER TABLE comments RENAME TO comments_previous'); + await database.exec(`CREATE TABLE comments ( + id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL REFERENCES comment_threads(id) ON DELETE CASCADE, + author_id TEXT NOT NULL REFERENCES users(id), + parent_id TEXT REFERENCES comments(id), + body_markdown TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending_review' CHECK(status IN ('pending_review', 'visible', 'removed')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + removed_at TEXT, + removed_by TEXT REFERENCES users(id), + removal_reason TEXT + )`); + await database.exec(`INSERT INTO comments( + id,thread_id,author_id,parent_id,body_markdown,status,created_at,updated_at, + removed_at,removed_by,removal_reason + ) + SELECT id,thread_id,author_id,parent_id,body_markdown,status,created_at,updated_at, + removed_at,removed_by,removal_reason + FROM comments_previous`); + await database.exec('DROP TABLE comments_previous'); + await database.exec('COMMIT'); + } catch (error) { + await database.exec('ROLLBACK'); + throw error; + } finally { + await database.exec('PRAGMA foreign_keys=ON'); + } +} + +const moderationScansDefinition = await (await database.prepare( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='content_moderation_scans'" +)).get(); +const moderationScansMissingEvidence = ( + typeof moderationScansDefinition?.sql === 'string' + && ( + !moderationScansDefinition.sql.includes('author_id TEXT') + || !moderationScansDefinition.sql.includes('risk_level TEXT') + || !moderationScansDefinition.sql.includes('reason TEXT') + ) +); +if ( + typeof moderationScansDefinition?.sql === 'string' + && ( + !moderationScansDefinition.sql.includes("'avatar'") + || moderationScansMissingEvidence + ) +) { + if (moderationScansMissingEvidence) { + const existingScans = await ( + await database.prepare('SELECT COUNT(*) AS count FROM content_moderation_scans') + ).get(); + if (Number(existingScans?.count) > 0) { + throw new Error( + 'Existing moderation scans lack required author, risk, or reason evidence and cannot be backfilled safely.' + ); + } + } + console.log('Aligning automated community moderation scan evidence.'); + await database.exec('PRAGMA foreign_keys=OFF'); + await database.exec('BEGIN'); + try { + await database.exec(`CREATE TABLE content_moderation_scans_expanded ( + id TEXT PRIMARY KEY, + content_type TEXT NOT NULL CHECK(content_type IN ('comment','profile_bio','guide','avatar')), + content_id TEXT NOT NULL, + author_id TEXT NOT NULL REFERENCES users(id), + content_hash TEXT NOT NULL CHECK(length(content_hash)>0), + revision_id TEXT REFERENCES moderated_content_revisions(id), + policy_version INTEGER NOT NULL CHECK(policy_version>=1), + provider TEXT NOT NULL CHECK(length(trim(provider))>0), + model TEXT NOT NULL CHECK(length(trim(model))>0), + provider_request_id TEXT, + status TEXT NOT NULL CHECK(status IN ('approved','review','priority_review','error')), + risk_level TEXT NOT NULL CHECK(risk_level IN ('low','medium','high')), + reason TEXT NOT NULL CHECK(length(trim(reason))>0), + scores_json TEXT NOT NULL DEFAULT '{}', + flags_json TEXT NOT NULL DEFAULT '[]', + rule_matches_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + completed_at TEXT, + error_code TEXT, + error_message TEXT, + UNIQUE(content_type,content_id,content_hash,policy_version,provider,model), + CHECK( + (content_type IN ('comment','avatar') AND revision_id IS NULL) + OR (content_type IN ('profile_bio','guide') AND revision_id IS NOT NULL) + ), + CHECK( + (status='error' AND error_code IS NOT NULL) + OR (status<>'error' AND error_code IS NULL AND error_message IS NULL) + ) + )`); + if (!moderationScansMissingEvidence) { + await database.exec(`INSERT INTO content_moderation_scans_expanded( + id,content_type,content_id,author_id,content_hash,revision_id,policy_version,provider,model, + provider_request_id,status,risk_level,reason,scores_json,flags_json,rule_matches_json, + created_at,completed_at,error_code,error_message + ) + SELECT id,content_type,content_id,author_id,content_hash,revision_id,policy_version,provider,model, + provider_request_id,status,risk_level,reason,scores_json,flags_json,rule_matches_json, + created_at,completed_at,error_code,error_message + FROM content_moderation_scans`); + } + await database.exec('DROP TABLE content_moderation_scans'); + await database.exec('ALTER TABLE content_moderation_scans_expanded RENAME TO content_moderation_scans'); + await database.exec('COMMIT'); + } catch (error) { + await database.exec('ROLLBACK'); + throw error; + } finally { + await database.exec('PRAGMA foreign_keys=ON'); + } +} + +const communityModerationSettingsColumns = await ( + await database.prepare("PRAGMA table_info('community_moderation_settings')") +).all(); +if ( + communityModerationSettingsColumns.length + && !communityModerationSettingsColumns.some((column) => column.name === 'scan_avatars') +) { + console.log('Adding avatar scanning to community moderation settings.'); + await database.exec(`ALTER TABLE community_moderation_settings + ADD COLUMN scan_avatars INTEGER NOT NULL DEFAULT 1 CHECK(scan_avatars IN (0,1))`); +} + +const avatarReviewColumns = await ( + await database.prepare("PRAGMA table_info('avatar_reviews')") +).all(); +if ( + avatarReviewColumns.length + && !avatarReviewColumns.some((column) => column.name === 'review_origin') +) { + console.log('Adding review origin evidence to avatar oversight.'); + await database.exec(`ALTER TABLE avatar_reviews + ADD COLUMN review_origin TEXT NOT NULL DEFAULT 'legacy' + CHECK(review_origin IN ('automated','user_report','legacy'))`); +} + // The project schema deliberately avoids semicolons inside string literals. const statements = schema .split(';') diff --git a/src/lib/components/ContentReport.svelte b/src/lib/components/ContentReport.svelte new file mode 100644 index 0000000..2d4bed4 --- /dev/null +++ b/src/lib/components/ContentReport.svelte @@ -0,0 +1,63 @@ + + +
+ {label} +
+ REQUEST OVERSIGHT REVIEW +

Send this content to Community Oversight. A report does not remove it automatically.

+ + {#snippet children(pending)} + + + + + {#if matchesResult && result?.userReportError}

{String(result.userReportError)}

{/if} + {#if matchesResult && result?.userReportSaved}

Report sent to Community Oversight.

{/if} +
+ +
+ {/snippet} + +
+
+ + diff --git a/src/lib/components/ModerationDisclosure.svelte b/src/lib/components/ModerationDisclosure.svelte new file mode 100644 index 0000000..085c309 --- /dev/null +++ b/src/lib/components/ModerationDisclosure.svelte @@ -0,0 +1,17 @@ + + + diff --git a/src/lib/server/avatars.ts b/src/lib/server/avatars.ts index d236a68..7e2f56a 100644 --- a/src/lib/server/avatars.ts +++ b/src/lib/server/avatars.ts @@ -49,7 +49,12 @@ export async function processAvatarUpload(file: File): Promise { } } -export async function saveUploadedAvatar(database: Database, userId: string, output: Buffer): Promise { +export async function saveUploadedAvatar( + database: Database, + userId: string, + output: Buffer, + options: { queueReview: boolean } = { queueReview: true } +): Promise<{ uploadedAt: string; reviewId: string | null }> { const key = storageKey(userId); const directory = avatarStorageDirectory(); await mkdir(directory, { recursive: true }); @@ -61,18 +66,55 @@ export async function saveUploadedAvatar(database: Database, userId: string, out .run(userId, 'upload', key, output.length, now); await (await database.prepare("UPDATE avatar_reviews SET status='superseded',reviewed_at=? WHERE user_id=? AND status='pending'")) .run(now, userId); + let reviewId: string | null = null; + if (options.queueReview) { + reviewId = await queueCurrentAvatarReview( + database, + userId, + { + origin: 'automated', + summary: 'Automated safety checks flagged a custom avatar for Oversight review.' + } + ); + } + return { uploadedAt: now, reviewId }; +} + +export async function queueCurrentAvatarReview( + database: Database, + userId: string, + options: { + origin: 'automated' | 'user_report' | 'legacy'; + summary?: string; + } = { origin: 'user_report' } +): Promise { + const avatar = await (await database.prepare(`SELECT source,value,byte_size,updated_at + FROM user_avatars WHERE user_id=?`)).get(userId); + if (avatar?.source !== 'upload' || typeof avatar.value !== 'string') return null; + const existing = await (await database.prepare( + "SELECT id FROM avatar_reviews WHERE user_id=? AND avatar_updated_at=? AND status='pending'" + )).get(userId, avatar.updated_at); + if (existing?.id) return String(existing.id); const reviewId = randomUUID(); await (await database.prepare(`INSERT INTO avatar_reviews - (id,user_id,avatar_updated_at,storage_key,byte_size,status,submitted_at) - VALUES(?,?,?,?,?,'pending',?)`)).run(reviewId, userId, now, key, output.length, now); + (id,user_id,avatar_updated_at,storage_key,byte_size,status,review_origin,submitted_at) + VALUES(?,?,?,?,?,'pending',?,?)`)).run( + reviewId, + userId, + avatar.updated_at, + avatar.value, + avatar.byte_size, + options.origin, + new Date().toISOString() + ); await notifyOversightReview({ category: 'avatars', itemId: reviewId, title: 'Avatar awaiting review', - summary: 'A new custom agent avatar was uploaded and is ready for Oversight review.', + summary: options.summary ?? 'An Agent report requested Oversight review of a custom avatar.', link: '/isac/avatar-oversight?status=pending' }).catch((error) => console.error('Unable to notify Avatar Oversight reviewers', error)); - return now; + return reviewId; } export async function selectPresetAvatar(database: Database, userId: string, preset: string): Promise { diff --git a/src/lib/server/community-moderation.test.ts b/src/lib/server/community-moderation.test.ts new file mode 100644 index 0000000..8d77a12 --- /dev/null +++ b/src/lib/server/community-moderation.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { + assessCommunityModeration, + findLocalModerationMatches, + normalizeModerationTerm, + type CommunityModerationPolicy +} from './community-moderation'; + +const policy = (overrides: Partial = {}): CommunityModerationPolicy => ({ + automatedEnabled: true, + autoPublishClean: true, + holdOnProviderError: true, + scanComments: true, + scanProfiles: true, + scanGuides: true, + scanAvatars: true, + providerModel: 'omni-moderation-latest', + detectPii: true, + maxLinks: 1, + duplicateWindowSeconds: 300, + policyVersion: 1, + thresholds: [{ + category: 'harassment', + enabled: true, + reviewThreshold: 0.5, + priorityThreshold: 0.85 + }], + terms: [], + ...overrides +}); + +describe('community moderation policy', () => { + it('normalizes configured terms consistently', () => { + expect(normalizeModerationTerm(' Rogue\u00a0Agent ')).toBe('rogue agent'); + }); + + it('allows an exact local exception to suppress a matching banned term', () => { + const matches = findLocalModerationMatches('Rogue Agent tactics', 'comment', policy({ + detectPii: false, + terms: [ + { id: 'ban', listType: 'ban', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true }, + { id: 'allow', listType: 'allow', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true } + ] + })); + expect(matches).toEqual([]); + }); + + it('finds scoped banned terms, PII, and excessive links', () => { + const matches = findLocalModerationMatches( + 'Contact test@example.com about forbidden plans at https://one.test and https://two.test', + 'comment', + policy({ + terms: [{ + id: 'ban', + listType: 'ban', + matchMode: 'word', + value: 'forbidden', + normalizedValue: 'forbidden', + scopes: ['comment'], + enabled: true + }] + }) + ); + expect(matches.map((match) => match.kind)).toEqual(['banned_term', 'pii', 'link_limit']); + }); + + it('routes priority scores ahead of ordinary review scores', () => { + expect(assessCommunityModeration({ + policy: policy(), + contentType: 'comment', + providerFlagged: true, + categoryScores: { harassment: 0.9 }, + ruleMatches: [] + }).decision).toBe('priority_review'); + }); + + it('holds provider failures under the default fail-closed policy', () => { + expect(assessCommunityModeration({ + policy: policy(), + contentType: 'profile_bio', + providerFlagged: false, + categoryScores: {}, + ruleMatches: [], + providerError: 'timeout' + }).decision).toBe('error'); + }); + + it('publishes clean content only when auto publication is enabled', () => { + expect(assessCommunityModeration({ + policy: policy(), + contentType: 'comment', + providerFlagged: false, + categoryScores: { harassment: 0.01 }, + ruleMatches: [] + }).decision).toBe('approved'); + expect(assessCommunityModeration({ + policy: policy({ autoPublishClean: false }), + contentType: 'comment', + providerFlagged: false, + categoryScores: { harassment: 0.01 }, + ruleMatches: [] + }).decision).toBe('review'); + }); + + it('publishes locally clean content when provider scanning is disabled', () => { + expect(assessCommunityModeration({ + policy: policy({ automatedEnabled: false }), + contentType: 'comment', + providerFlagged: false, + categoryScores: {}, + ruleMatches: [] + }).decision).toBe('approved'); + }); +}); diff --git a/src/lib/server/community-moderation.ts b/src/lib/server/community-moderation.ts new file mode 100644 index 0000000..46739ad --- /dev/null +++ b/src/lib/server/community-moderation.ts @@ -0,0 +1,611 @@ +import { env } from '$env/dynamic/private'; +import { getDatabase } from '$lib/server/database'; +import { createHash, randomUUID } from 'node:crypto'; +import OpenAI from 'openai'; + +export const MODERATION_CATEGORIES = [ + 'harassment', + 'harassment/threatening', + 'hate', + 'hate/threatening', + 'illicit', + 'illicit/violent', + 'self-harm', + 'self-harm/intent', + 'self-harm/instructions', + 'sexual', + 'sexual/minors', + 'violence', + 'violence/graphic' +] as const; + +export type ModerationCategory = (typeof MODERATION_CATEGORIES)[number]; +export type ModeratedContentType = 'comment' | 'profile_bio' | 'guide' | 'avatar'; +export type ModerationDecision = 'approved' | 'review' | 'priority_review' | 'error'; + +export type ModerationThreshold = { + category: ModerationCategory; + enabled: boolean; + reviewThreshold: number; + priorityThreshold: number; +}; + +export type ModerationTerm = { + id: string; + listType: 'allow' | 'ban'; + matchMode: 'word' | 'phrase'; + value: string; + normalizedValue: string; + scopes: ModeratedContentType[]; + enabled: boolean; +}; + +export type CommunityModerationPolicy = { + automatedEnabled: boolean; + autoPublishClean: boolean; + holdOnProviderError: boolean; + scanComments: boolean; + scanProfiles: boolean; + scanGuides: boolean; + scanAvatars: boolean; + providerModel: 'omni-moderation-latest' | 'omni-moderation-2024-09-26'; + detectPii: boolean; + maxLinks: number; + duplicateWindowSeconds: number; + policyVersion: number; + thresholds: ModerationThreshold[]; + terms: ModerationTerm[]; +}; + +export type LocalModerationMatch = { + kind: 'banned_term' | 'pii' | 'link_limit'; + label: string; + excerpt: string; + termId?: string; +}; + +export type CommunityModerationResult = { + contentHash: string; + decision: ModerationDecision; + riskLevel: 'low' | 'medium' | 'high'; + reason: string; + policyVersion: number; + provider: 'openai' | 'local'; + model: string; + requestId: string | null; + categoryScores: Partial>; + categoryFlags: Partial>; + ruleMatches: LocalModerationMatch[]; + error: string | null; +}; + +const DEFAULT_POLICY: CommunityModerationPolicy = { + automatedEnabled: false, + autoPublishClean: true, + holdOnProviderError: true, + scanComments: true, + scanProfiles: true, + scanGuides: true, + scanAvatars: true, + providerModel: 'omni-moderation-latest', + detectPii: true, + maxLinks: 3, + duplicateWindowSeconds: 300, + policyVersion: 1, + thresholds: [], + terms: [] +}; + +let cachedPolicy: { value: CommunityModerationPolicy; expiresAt: number } | null = null; +let client: OpenAI | null = null; + +export function invalidateCommunityModerationPolicy() { + cachedPolicy = null; +} + +export function normalizeModerationTerm(value: string) { + return value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase(); +} + +export function hashModeratedContent(value: string) { + return createHash('sha256').update(value.normalize('NFC')).digest('hex'); +} + +function parseScopes(value: unknown): ModeratedContentType[] { + try { + const parsed = typeof value === 'string' ? JSON.parse(value) : value; + if (!Array.isArray(parsed)) return []; + return parsed.filter((scope): scope is ModeratedContentType => + scope === 'comment' || scope === 'profile_bio' || scope === 'guide' || scope === 'avatar' + ); + } catch { + return []; + } +} + +function boundedNumber(value: unknown, fallback: number, min: number, max: number) { + const number = Number(value); + return Number.isFinite(number) ? Math.min(max, Math.max(min, number)) : fallback; +} + +export async function getCommunityModerationPolicy(): Promise { + if (cachedPolicy && cachedPolicy.expiresAt > Date.now()) return cachedPolicy.value; + const database = await getDatabase(); + const [settings, thresholdRows, termRows] = await Promise.all([ + (await database.prepare("SELECT * FROM community_moderation_settings WHERE id='primary'")).get(), + (await database.prepare('SELECT * FROM community_moderation_thresholds ORDER BY category')).all(), + (await database.prepare('SELECT * FROM community_moderation_terms WHERE enabled=1 ORDER BY list_kind,value')).all() + ]); + const model = String(settings?.provider_model ?? DEFAULT_POLICY.providerModel); + const value: CommunityModerationPolicy = { + automatedEnabled: Boolean(settings?.automated_enabled), + autoPublishClean: settings?.auto_publish_clean === undefined + ? DEFAULT_POLICY.autoPublishClean + : Boolean(settings.auto_publish_clean), + holdOnProviderError: settings?.hold_on_provider_error === undefined + ? DEFAULT_POLICY.holdOnProviderError + : Boolean(settings.hold_on_provider_error), + scanComments: settings?.scan_comments === undefined ? true : Boolean(settings.scan_comments), + scanProfiles: settings?.scan_profiles === undefined ? true : Boolean(settings.scan_profiles), + scanGuides: settings?.scan_guides === undefined ? true : Boolean(settings.scan_guides), + scanAvatars: settings?.scan_avatars === undefined ? true : Boolean(settings.scan_avatars), + providerModel: model === 'omni-moderation-2024-09-26' + ? model + : 'omni-moderation-latest', + detectPii: settings?.detect_pii === undefined ? true : Boolean(settings.detect_pii), + maxLinks: boundedNumber(settings?.max_links, DEFAULT_POLICY.maxLinks, 0, 20), + duplicateWindowSeconds: boundedNumber( + settings?.duplicate_window_seconds, + DEFAULT_POLICY.duplicateWindowSeconds, + 0, + 3_600 + ), + policyVersion: Math.max(1, Number(settings?.policy_version ?? 1)), + thresholds: thresholdRows.flatMap((row) => { + const category = String(row.category); + if (!MODERATION_CATEGORIES.includes(category as ModerationCategory)) return []; + return [{ + category: category as ModerationCategory, + enabled: Boolean(row.enabled), + reviewThreshold: boundedNumber(row.review_threshold, 0.5, 0, 1), + priorityThreshold: boundedNumber(row.priority_threshold, 0.85, 0, 1) + }]; + }), + terms: termRows.flatMap((row) => { + const listType = String(row.list_kind); + const matchMode = String(row.match_mode); + if (!['allow', 'ban'].includes(listType) || !['word', 'phrase'].includes(matchMode)) return []; + return [{ + id: String(row.id), + listType: listType as 'allow' | 'ban', + matchMode: matchMode as 'word' | 'phrase', + value: String(row.value), + normalizedValue: String(row.normalized_value), + scopes: parseScopes(row.scopes_json), + enabled: Boolean(row.enabled) + }]; + }) + }; + cachedPolicy = { value, expiresAt: Date.now() + 30_000 }; + return value; +} + +function scopeEnabled(policy: CommunityModerationPolicy, contentType: ModeratedContentType) { + if (contentType === 'comment') return policy.scanComments; + if (contentType === 'profile_bio') return policy.scanProfiles; + if (contentType === 'guide') return policy.scanGuides; + return policy.scanAvatars; +} + +function wordPattern(value: string) { + const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`(^|[^\\p{L}\\p{N}_])${escaped}(?=$|[^\\p{L}\\p{N}_])`, 'iu'); +} + +function matchesTerm(text: string, term: ModerationTerm) { + return term.matchMode === 'word' + ? wordPattern(term.normalizedValue).test(text) + : text.includes(term.normalizedValue); +} + +function excerpt(value: string, limit = 80) { + const compact = value.replace(/\s+/g, ' ').trim(); + return compact.length > limit ? `${compact.slice(0, limit - 1)}…` : compact; +} + +const piiPatterns = [ + { label: 'EMAIL ADDRESS', pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/iu }, + { label: 'PHONE NUMBER', pattern: /(?:\+?\d[\d\s().-]{7,}\d)/u }, + { label: 'IP ADDRESS', pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/u } +] as const; + +const linkPattern = /\bhttps?:\/\/[^\s<>()]+/giu; + +export function findLocalModerationMatches( + text: string, + contentType: ModeratedContentType, + policy: Pick +): LocalModerationMatch[] { + const normalized = normalizeModerationTerm(text); + const scoped = policy.terms.filter((term) => + term.enabled && (term.scopes.length === 0 || term.scopes.includes(contentType)) + ); + const allowKeys = new Set(scoped.filter((term) => term.listType === 'allow' && matchesTerm(normalized, term)) + .map((term) => `${term.matchMode}:${term.normalizedValue}`)); + const matches: LocalModerationMatch[] = scoped + .filter((term) => + term.listType === 'ban' + && !allowKeys.has(`${term.matchMode}:${term.normalizedValue}`) + && matchesTerm(normalized, term) + ) + .map((term) => ({ + kind: 'banned_term', + label: `BANNED ${term.matchMode.toLocaleUpperCase()}`, + excerpt: excerpt(term.value), + termId: term.id + })); + if (policy.detectPii) { + for (const definition of piiPatterns) { + const match = text.match(definition.pattern); + if (match?.[0]) matches.push({ + kind: 'pii', + label: definition.label, + excerpt: excerpt(match[0]) + }); + } + } + const links = [...text.matchAll(linkPattern)]; + if (links.length > policy.maxLinks) matches.push({ + kind: 'link_limit', + label: 'LINK LIMIT', + excerpt: `${links.length} links submitted; policy allows ${policy.maxLinks}` + }); + return matches; +} + +export function assessCommunityModeration(input: { + policy: CommunityModerationPolicy; + contentType: ModeratedContentType; + providerFlagged: boolean; + categoryScores: Partial>; + categoryFlags?: Partial>; + ruleMatches: LocalModerationMatch[]; + providerError?: string | null; +}): Pick { + if (!input.policy.automatedEnabled || !scopeEnabled(input.policy, input.contentType)) { + if (input.ruleMatches.some((match) => match.kind === 'banned_term')) { + return { decision: 'priority_review', riskLevel: 'high', reason: 'A configured banned term requires priority review.' }; + } + if (input.ruleMatches.length) { + return { decision: 'review', riskLevel: 'medium', reason: 'A deterministic safety rule requires review.' }; + } + return { decision: 'approved', riskLevel: 'low', reason: 'Automated provider scanning is disabled and no local rule was triggered.' }; + } + if (input.providerError) { + return input.policy.holdOnProviderError + ? { decision: 'error', riskLevel: 'medium', reason: 'The moderation provider could not classify this content.' } + : { decision: 'approved', riskLevel: 'low', reason: 'Provider errors are configured to allow publication.' }; + } + if (input.ruleMatches.some((match) => match.kind === 'banned_term')) { + return { decision: 'priority_review', riskLevel: 'high', reason: 'A configured banned term requires priority review.' }; + } + if (input.ruleMatches.length) { + return { decision: 'review', riskLevel: 'medium', reason: 'A deterministic safety rule requires review.' }; + } + const enabledThresholds = input.policy.thresholds.filter((threshold) => threshold.enabled); + if (enabledThresholds.some((threshold) => + Number(input.categoryScores[threshold.category] ?? 0) >= threshold.priorityThreshold + )) { + return { decision: 'priority_review', riskLevel: 'high', reason: 'A provider category exceeded its priority threshold.' }; + } + if (input.providerFlagged || enabledThresholds.some((threshold) => + Number(input.categoryScores[threshold.category] ?? 0) >= threshold.reviewThreshold + )) { + return { decision: 'review', riskLevel: 'medium', reason: 'The moderation provider flagged this content for review.' }; + } + return input.policy.autoPublishClean + ? { decision: 'approved', riskLevel: 'low', reason: 'No configured moderation rule was triggered.' } + : { decision: 'review', riskLevel: 'medium', reason: 'Clean content requires human approval under the current policy.' }; +} + +function moderationClient() { + const apiKey = env.OPENAI_MODERATION_API_KEY?.trim() || env.OPENAI_MECHANIC_API_KEY?.trim(); + if (!apiKey) return null; + if (!client) client = new OpenAI({ apiKey, timeout: 15_000, maxRetries: 1 }); + return client; +} + +export async function moderateCommunityText(input: { + contentType: ModeratedContentType; + text: string; + policy?: CommunityModerationPolicy; +}): Promise { + const policy = input.policy ?? await getCommunityModerationPolicy(); + const contentHash = hashModeratedContent(input.text); + const ruleMatches = findLocalModerationMatches(input.text, input.contentType, policy); + if (!policy.automatedEnabled || !scopeEnabled(policy, input.contentType)) { + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: false, + categoryScores: {}, + ruleMatches + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'local', + model: 'local-rules-v1', + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches, + error: null + }; + } + const openai = moderationClient(); + if (!openai) { + const error = 'OpenAI moderation is not configured.'; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: false, + categoryScores: {}, + ruleMatches, + providerError: error + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: policy.providerModel, + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches, + error + }; + } + try { + const response = await openai.moderations.create({ + model: policy.providerModel, + input: input.text + }); + const result = response.results[0]; + const scores = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [ + category, + Number(result?.category_scores[category] ?? 0) + ])) as Record; + const flags = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [ + category, + Boolean(result?.categories[category]) + ])) as Record; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: Boolean(result?.flagged), + categoryScores: scores, + categoryFlags: flags, + ruleMatches + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: String(response.model), + requestId: String((response as typeof response & { _request_id?: string })._request_id ?? response.id), + categoryScores: scores, + categoryFlags: flags, + ruleMatches, + error: null + }; + } catch (error) { + const message = error instanceof Error ? error.message.slice(0, 500) : 'Moderation provider request failed.'; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: false, + categoryScores: {}, + ruleMatches, + providerError: message + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: policy.providerModel, + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches, + error: message + }; + } +} + +export async function moderateCommunityImage(input: { + contentType: 'avatar'; + image: Buffer; + policy?: CommunityModerationPolicy; +}): Promise { + const policy = input.policy ?? await getCommunityModerationPolicy(); + const contentHash = createHash('sha256').update(input.image).digest('hex'); + if (!policy.automatedEnabled || !scopeEnabled(policy, input.contentType)) { + return { + contentHash, + decision: 'approved', + riskLevel: 'low', + reason: 'Automated image scanning is disabled.', + policyVersion: policy.policyVersion, + provider: 'local', + model: 'local-rules-v1', + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches: [], + error: null + }; + } + const openai = moderationClient(); + if (!openai) { + const error = 'OpenAI moderation is not configured.'; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: false, + categoryScores: {}, + ruleMatches: [], + providerError: error + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: policy.providerModel, + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches: [], + error + }; + } + try { + const response = await openai.moderations.create({ + model: policy.providerModel, + input: [{ + type: 'image_url', + image_url: { url: `data:image/webp;base64,${input.image.toString('base64')}` } + }] + }); + const result = response.results[0]; + const scores = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [ + category, + Number(result?.category_scores[category] ?? 0) + ])) as Record; + const flags = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [ + category, + Boolean(result?.categories[category]) + ])) as Record; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: Boolean(result?.flagged), + categoryScores: scores, + categoryFlags: flags, + ruleMatches: [] + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: String(response.model), + requestId: String((response as typeof response & { _request_id?: string })._request_id ?? response.id), + categoryScores: scores, + categoryFlags: flags, + ruleMatches: [], + error: null + }; + } catch (error) { + const message = error instanceof Error ? error.message.slice(0, 500) : 'Moderation provider request failed.'; + const assessment = assessCommunityModeration({ + policy, + contentType: input.contentType, + providerFlagged: false, + categoryScores: {}, + ruleMatches: [], + providerError: message + }); + return { + ...assessment, + contentHash, + policyVersion: policy.policyVersion, + provider: 'openai', + model: policy.providerModel, + requestId: null, + categoryScores: {}, + categoryFlags: {}, + ruleMatches: [], + error: message + }; + } +} + +export async function recordCommunityModerationScan(input: { + contentType: ModeratedContentType; + contentId: string; + revisionId?: string | null; + authorId: string; + result: CommunityModerationResult; +}) { + const database = await getDatabase(); + const id = randomUUID(); + const now = new Date().toISOString(); + const result = await (await database.prepare(`INSERT OR IGNORE INTO content_moderation_scans + (id,content_type,content_id,revision_id,author_id,content_hash,policy_version, + provider,model,provider_request_id,status,risk_level,reason,scores_json, + flags_json,rule_matches_json,error_code,error_message,created_at,completed_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)).run( + id, + input.contentType, + input.contentId, + input.revisionId ?? null, + input.authorId, + input.result.contentHash, + input.result.policyVersion, + input.result.provider, + input.result.model, + input.result.requestId, + input.result.decision, + input.result.riskLevel, + input.result.reason, + JSON.stringify(input.result.categoryScores), + JSON.stringify(input.result.categoryFlags), + JSON.stringify(input.result.ruleMatches), + input.result.error ? 'provider_error' : null, + input.result.error, + now, + now + ); + if (result.changes) return id; + const existing = await (await database.prepare(`SELECT id FROM content_moderation_scans + WHERE content_type=? AND content_id=? AND content_hash=? AND policy_version=? + AND provider=? AND model=?`)).get( + input.contentType, + input.contentId, + input.result.contentHash, + input.result.policyVersion, + input.result.provider, + input.result.model + ); + return String(existing?.id ?? id); +} + +export async function createUserContentReport(input: { + contentType: ModeratedContentType; + contentId: string; + reporterId: string; + category: 'spam' | 'harassment' | 'hate' | 'sexual' | 'violence' | 'self_harm' | 'privacy' | 'other'; + details?: string | null; +}) { + const database = await getDatabase(); + const id = randomUUID(); + const createdAt = new Date().toISOString(); + const result = await (await database.prepare(`INSERT OR IGNORE INTO user_content_reports + (id,content_type,content_id,reporter_id,category,details,status,created_at) + VALUES(?,?,?,?,?,?,'pending',?)`)).run( + id, + input.contentType, + input.contentId, + input.reporterId, + input.category, + input.details?.trim() || null, + createdAt + ); + return result.changes ? { id, createdAt } : null; +} diff --git a/src/routes/account/+page.server.ts b/src/routes/account/+page.server.ts index ce70f68..f613128 100644 --- a/src/routes/account/+page.server.ts +++ b/src/routes/account/+page.server.ts @@ -11,12 +11,17 @@ import { canRemoveAuthenticationMethod, getAuthenticationMethods } from '$lib/se import { recordIsacAudit } from '$lib/server/audit'; import { normalizeUserMarkdown, renderUserMarkdown } from '$lib/markdown'; import { randomUUID } from 'node:crypto'; +import { + moderateCommunityImage, + moderateCommunityText, + recordCommunityModerationScan +} from '$lib/server/community-moderation'; export const load: PageServerLoad = async ({ cookies, url }) => { const user = await getSessionUser(cookies); if (!user) redirect(303, '/sign-in'); const database = await getDatabase(); - const [builds, guides, favorites, ratings, connection, emailIdentity, passkeys, authenticationMethods, notifications, notificationUnread, profile] = await Promise.all([ + const [builds, guides, favorites, ratings, connection, emailIdentity, passkeys, authenticationMethods, notifications, notificationUnread, profile, pendingBioRevision] = await Promise.all([ (await database.prepare('SELECT id, title, slug, visibility, updated_at FROM builds WHERE owner_id = ? ORDER BY updated_at DESC')).all(user.id), (await database.prepare("SELECT id, title, slug, visibility, updated_at FROM builds WHERE owner_id = ? AND guide_body IS NOT NULL AND length(trim(guide_body)) > 0 ORDER BY updated_at DESC")).all(user.id), (await database.prepare('SELECT builds.id, builds.title, builds.slug, COALESCE(user_profiles.display_name,users.username) AS author FROM favorites JOIN builds ON builds.id = favorites.build_id JOIN users ON users.id = builds.owner_id LEFT JOIN user_profiles ON user_profiles.user_id=users.id WHERE favorites.user_id = ? ORDER BY favorites.created_at DESC')).all(user.id), @@ -38,7 +43,14 @@ export const load: PageServerLoad = async ({ cookies, url }) => { COALESCE(moderation.bio_hidden,0) AS bio_hidden,moderation.reason AS moderation_reason FROM user_profiles profile LEFT JOIN profile_moderation moderation ON moderation.user_id=profile.user_id - WHERE profile.user_id=?`)).get(user.id) + WHERE profile.user_id=?`)).get(user.id), + (await database.prepare(`SELECT revision.body_markdown,revision.created_at,scan.status,scan.reason + FROM moderated_content_revisions revision + JOIN content_moderation_scans scan ON scan.revision_id=revision.id + LEFT JOIN content_moderation_reviews review ON review.scan_id=scan.id + WHERE revision.content_type='profile_bio' AND revision.content_id=? + AND scan.status IN ('review','priority_review','error') AND review.id IS NULL + ORDER BY revision.created_at DESC LIMIT 1`)).get(user.id) ]); const oauthErrorCode = url.searchParams.get('oauth_error'); const oauthErrors: Record = { @@ -64,7 +76,7 @@ export const load: PageServerLoad = async ({ cookies, url }) => { return { user: { ...user, - bioMarkdown: String(profile?.bio_markdown ?? ''), + bioMarkdown: String(pendingBioRevision?.body_markdown ?? profile?.bio_markdown ?? ''), profileIsPublic: profile?.is_public === undefined ? true : Boolean(profile.is_public) }, account: { builds, guides, favorites, ratings, notifications, notificationUnread: Number(notificationUnread?.count ?? 0) }, @@ -96,6 +108,9 @@ export const load: PageServerLoad = async ({ cookies, url }) => { isPublic: Boolean(profile?.is_public), bioHidden: Boolean(profile?.bio_hidden), moderationReason: profile?.moderation_reason ? String(profile.moderation_reason) : null, + bioReviewPending: Boolean(pendingBioRevision), + bioReviewReason: pendingBioRevision?.reason ? String(pendingBioRevision.reason) : null, + pendingBioSubmittedAt: pendingBioRevision?.created_at ? String(pendingBioRevision.created_at) : null, bioUpdatedAt: profile?.bio_updated_at ? String(profile.bio_updated_at) : null, href: `/agents/${encodeURIComponent(user.id)}` } @@ -116,16 +131,37 @@ export const actions: Actions = { } const now = new Date().toISOString(); const database = await getDatabase(); - await (await database.prepare(`INSERT INTO user_profiles - (user_id,display_name,bio_markdown,is_public,bio_updated_at,updated_at) - VALUES(?,?,?,1,?,?) - ON CONFLICT(user_id) DO UPDATE SET - bio_markdown=excluded.bio_markdown, - bio_updated_at=excluded.bio_updated_at, - updated_at=excluded.updated_at`)) - .run(user.id, user.displayName, bio, now, now); + const moderation = await moderateCommunityText({ contentType: 'profile_bio', text: bio }); + const candidateRevisionId = randomUUID(); + await (await database.prepare(`INSERT OR IGNORE INTO moderated_content_revisions + (id,content_type,content_id,submitted_by,body_markdown,content_hash,created_at) + VALUES(?,'profile_bio',?,?,?,?,?)`)) + .run(candidateRevisionId, user.id, user.id, bio, moderation.contentHash, now); + const storedRevision = await (await database.prepare(`SELECT id + FROM moderated_content_revisions + WHERE content_type='profile_bio' AND content_id=? AND content_hash=?`)) + .get(user.id, moderation.contentHash); + const revisionId = String(storedRevision?.id ?? candidateRevisionId); + if (moderation.decision === 'approved') { + await (await database.prepare(`INSERT INTO user_profiles + (user_id,display_name,bio_markdown,is_public,bio_updated_at,updated_at) + VALUES(?,?,?,1,?,?) + ON CONFLICT(user_id) DO UPDATE SET + bio_markdown=excluded.bio_markdown, + bio_updated_at=excluded.bio_updated_at, + updated_at=excluded.updated_at`)) + .run(user.id, user.displayName, bio, now, now); + } + await recordCommunityModerationScan({ + contentType: 'profile_bio', + contentId: user.id, + revisionId, + authorId: user.id, + result: moderation + }); return { bioSaved: true, + bioReviewQueued: moderation.decision !== 'approved', bioMarkdown: bio, bioResultToken: randomUUID() }; @@ -227,10 +263,29 @@ export const actions: Actions = { if (!(file instanceof File)) return fail(400, { avatarError: 'Choose an image to upload.' }); try { const output = await processAvatarUpload(file); + const moderation = await moderateCommunityImage({ contentType: 'avatar', image: output }); const database = await getDatabase(); - const uploadedAt = await saveUploadedAvatar(database, user.id, output); - await recordIsacAudit({ actorId: user.id, category: 'user_moderation', action: 'avatar.submitted', entityId: user.id, after: { status: 'pending', uploadedAt, byteSize: output.length } }); - return { avatarChanged: true, avatarReviewQueued: true }; + const queueReview = moderation.decision !== 'approved'; + const { uploadedAt, reviewId } = await saveUploadedAvatar(database, user.id, output, { queueReview }); + await recordCommunityModerationScan({ + contentType: 'avatar', + contentId: user.id, + authorId: user.id, + result: moderation + }); + await recordIsacAudit({ + actorId: user.id, + category: 'user_moderation', + action: 'avatar.submitted', + entityId: user.id, + after: { + status: queueReview ? 'pending_review' : 'automatically_approved', + uploadedAt, + byteSize: output.length, + reviewId + } + }); + return { avatarChanged: true, avatarReviewQueued: queueReview }; } catch (error) { if (error instanceof AvatarValidationError) return fail(400, { avatarError: error.message }); throw error; diff --git a/src/routes/account/+page.svelte b/src/routes/account/+page.svelte index 828a592..0d1954c 100644 --- a/src/routes/account/+page.svelte +++ b/src/routes/account/+page.svelte @@ -3,6 +3,7 @@ import PendingForm from '$lib/components/PendingForm.svelte'; import ImageUploadInput from '$lib/components/ImageUploadInput.svelte'; import MarkdownEditor from '$lib/components/MarkdownEditor.svelte'; + import ModerationDisclosure from '$lib/components/ModerationDisclosure.svelte'; import { toast } from '$lib/toast'; import { browserSupportsWebAuthn, startRegistration } from '@simplewebauthn/browser'; import { onMount } from 'svelte'; @@ -60,8 +61,13 @@ if (!token || token === handledBioToken) return; handledBioToken = token; if (form?.bioSaved) { - bioDraft = String(user.bioMarkdown ?? bioDraft); - toast.success('Profile bio saved', 'Your public Agent profile has been updated.'); + bioDraft = String(form.bioMarkdown ?? user.bioMarkdown ?? bioDraft); + toast.success( + form?.bioReviewQueued ? 'Profile bio received' : 'Profile bio saved', + form?.bioReviewQueued + ? 'Automated checks sent this revision to Oversight. Your last approved bio remains public.' + : 'Your public Agent profile has been updated.' + ); } else if (form?.bioError) { toast.error('Profile bio not saved', String(form.bioError)); } @@ -154,8 +160,9 @@

CUSTOM AVATAR

JPEG, PNG, WebP, or AVIF · 5 MiB maximum · cropped to a 256 px square. + {#if form?.avatarError}

{form.avatarError}

{/if} - {#if form?.avatarChanged}

{form?.avatarReviewQueued ? 'Avatar active now · queued for Oversight review.' : 'Avatar updated.'}

{/if} + {#if form?.avatarChanged}

{form?.avatarReviewQueued ? 'Avatar received and held for Oversight review. The default portrait remains public.' : 'Avatar updated.'}

{/if} {/if} @@ -167,6 +174,9 @@ {#if data.profile.bioHidden} {/if} + {#if data.profile.bioReviewPending} + + {/if} + {#if form?.bioError}

{form.bioError}

{/if} - {#if form?.bioSaved}

Public bio updated.

{/if} + {#if form?.bioSaved}

{form?.bioReviewQueued ? 'Bio revision queued for Oversight review.' : 'Public bio updated.'}

{/if}
diff --git a/src/routes/agents/[id]/+page.server.ts b/src/routes/agents/[id]/+page.server.ts index 1aca398..6b29308 100644 --- a/src/routes/agents/[id]/+page.server.ts +++ b/src/routes/agents/[id]/+page.server.ts @@ -3,13 +3,19 @@ import { getSessionUser } from '$lib/server/auth/session'; import { getDatabase } from '$lib/server/database'; import { normalizeUserMarkdown, renderUserMarkdown } from '$lib/markdown'; import { createNotification } from '$lib/server/notifications'; +import { + getCommunityModerationPolicy, + moderateCommunityText, + createUserContentReport, + recordCommunityModerationScan +} from '$lib/server/community-moderation'; import { error, fail } from '@sveltejs/kit'; import { randomUUID } from 'node:crypto'; +import { queueCurrentAvatarReview } from '$lib/server/avatars'; import type { Actions, PageServerLoad } from './$types'; const COMMENT_MAX_CHARACTERS = 2_000; const RAPID_COMMENT_SECONDS = 10; -const DUPLICATE_COMMENT_SECONDS = 300; const unavailable = (row: Record | null | undefined) => { const banned = typeof row?.banned_until === 'string' && new Date(row.banned_until) > new Date(); @@ -163,11 +169,13 @@ export const load: PageServerLoad = async ({ params, cookies }) => { comment.avatar_updated_at ) }, - canDelete: viewer?.id === String(comment.author_id) + canDelete: viewer?.id === String(comment.author_id), + canReport: Boolean(viewer) && viewer?.id !== String(comment.author_id) })), viewer: { id: viewer?.id ?? null, canComment: Boolean(viewer) && Boolean(agent.is_public), + canReportProfile: Boolean(viewer) && !isSelf, isSelf } }; @@ -187,6 +195,7 @@ export const actions: Actions = { }); } const database = await getDatabase(); + const moderationPolicy = await getCommunityModerationPolicy(); const now = new Date(); const rows = await (await database.prepare(`SELECT users.id, COALESCE(profile.display_name,users.username) AS display_name,profile.is_public, @@ -207,7 +216,7 @@ export const actions: Actions = { const recent = await (await database.prepare(`SELECT comments.body_markdown,comments.created_at FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id - WHERE comments.author_id=? AND comments.status='visible' + WHERE comments.author_id=? AND comments.status IN ('visible','pending_review') AND thread.target_type='profile' AND thread.target_id=? ORDER BY comments.created_at DESC LIMIT 1`)).get(commenter.id, params.id); if (typeof recent?.created_at === 'string') { @@ -215,11 +224,17 @@ export const actions: Actions = { if (elapsed < RAPID_COMMENT_SECONDS) { return fail(429, { profileCommentError: 'Wait a few seconds before posting again.' }); } - if (elapsed < DUPLICATE_COMMENT_SECONDS && recent.body_markdown === body) { + if (elapsed < moderationPolicy.duplicateWindowSeconds && recent.body_markdown === body) { return fail(409, { profileCommentError: 'That comment was already posted.' }); } } + const moderation = await moderateCommunityText({ + contentType: 'comment', + text: body, + policy: moderationPolicy + }); + const commentStatus = moderation.decision === 'approved' ? 'visible' : 'pending_review'; const createdAt = now.toISOString(); const proposedThreadId = randomUUID(); await (await database.prepare(`INSERT OR IGNORE INTO comment_threads @@ -232,9 +247,15 @@ export const actions: Actions = { const commentId = randomUUID(); await (await database.prepare(`INSERT INTO comments (id,thread_id,author_id,parent_id,body_markdown,status,created_at,updated_at) - VALUES(?,?,?,NULL,?,'visible',?,?)`)) - .run(commentId, thread.id, commenter.id, body, createdAt, createdAt); - if (commenter.id !== params.id) { + VALUES(?,?,?,NULL,?,?,?,?)`)) + .run(commentId, thread.id, commenter.id, body, commentStatus, createdAt, createdAt); + await recordCommunityModerationScan({ + contentType: 'comment', + contentId: commentId, + authorId: commenter.id, + result: moderation + }); + if (commentStatus === 'visible' && commenter.id !== params.id) { await createNotification({ userId: params.id, category: 'comments', @@ -250,6 +271,7 @@ export const actions: Actions = { } return { profileCommentSaved: true, + profileCommentReviewQueued: commentStatus === 'pending_review', profileCommentId: commentId, profileCommentResultToken: randomUUID() }; @@ -278,5 +300,72 @@ export const actions: Actions = { profileCommentId: commentId, profileCommentResultToken: randomUUID() }; + }, + + reportContent: async ({ params, cookies, request }) => { + const reporter = await getSessionUser(cookies); + const form = await request.formData(); + const contentType = String(form.get('contentType') ?? ''); + const contentId = String(form.get('contentId') ?? '').trim(); + const category = String(form.get('category') ?? ''); + const details = String(form.get('details') ?? '').normalize('NFKC').trim(); + const failure = (status: number, message: string) => fail(status, { + userReportError: message, + userReportContentId: contentId, + userReportContentType: contentType, + userReportResultToken: randomUUID() + }); + if (!reporter) return failure(401, 'Sign in to request an Oversight review.'); + if (!['comment', 'profile_bio', 'avatar'].includes(contentType)) { + return failure(400, 'Choose reportable content.'); + } + if (!['spam', 'harassment', 'hate', 'sexual', 'violence', 'self_harm', 'privacy', 'other'].includes(category)) { + return failure(400, 'Choose the policy concern that best fits this report.'); + } + if (details.length > 500 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(details)) { + return failure(400, 'Report details may contain up to 500 visible characters.'); + } + const database = await getDatabase(); + if (contentType === 'comment') { + const comment = await (await database.prepare(`SELECT comments.author_id + FROM comments JOIN comment_threads thread ON thread.id=comments.thread_id + WHERE comments.id=? AND comments.status='visible' + AND thread.target_type='profile' AND thread.target_id=?`)).get(contentId, params.id); + if (!comment) return failure(404, 'That comment is no longer available.'); + if (String(comment.author_id) === reporter.id) return failure(400, 'You cannot report your own comment.'); + } else { + if (contentId !== params.id) return failure(400, 'That profile content does not match this Agent.'); + if (reporter.id === params.id) return failure(400, 'You cannot report your own profile content.'); + const profile = await (await database.prepare(`SELECT profile.bio_markdown,profile.is_public, + COALESCE(moderation.bio_hidden,0) AS bio_hidden,avatar.source AS avatar_source + FROM user_profiles profile + LEFT JOIN profile_moderation moderation ON moderation.user_id=profile.user_id + LEFT JOIN user_avatars avatar ON avatar.user_id=profile.user_id + WHERE profile.user_id=?`)).get(params.id); + if (!profile?.is_public) return failure(404, 'That profile is no longer available.'); + if (contentType === 'profile_bio' && (profile.bio_hidden || !String(profile.bio_markdown ?? '').trim())) { + return failure(404, 'That biography is no longer available.'); + } + if (contentType === 'avatar' && profile.avatar_source !== 'upload') { + return failure(400, 'Preset SHD avatars are managed site assets and do not need an Agent report.'); + } + } + const report = await createUserContentReport({ + contentType: contentType as 'comment' | 'profile_bio' | 'avatar', + contentId, + reporterId: reporter.id, + category: category as 'spam' | 'harassment' | 'hate' | 'sexual' | 'violence' | 'self_harm' | 'privacy' | 'other', + details + }); + if (!report) return failure(409, 'You already have a pending report for this content.'); + if (contentType === 'avatar') { + await queueCurrentAvatarReview(database, params.id, { origin: 'user_report' }); + } + return { + userReportSaved: true, + userReportContentId: contentId, + userReportContentType: contentType, + userReportResultToken: randomUUID() + }; } }; diff --git a/src/routes/agents/[id]/+page.svelte b/src/routes/agents/[id]/+page.svelte index e20a3cf..6df6c0e 100644 --- a/src/routes/agents/[id]/+page.svelte +++ b/src/routes/agents/[id]/+page.svelte @@ -1,7 +1,9 @@ Community Oversight — ISAC { if (event.key === 'Escape' && reviewing) closeModeration(); + if (event.key === 'Escape' && queueReviewing) closeQueueReview(); }}/>

COMMUNITY OVERSIGHT / PUBLIC CONTENT CONTROL

COMMUNITY OVERSIGHT

-

Review public discussion and Agent bios. Moderation preserves evidence, requires a reason, and remains reversible.

+

Resolve automated safety flags and community reports, then inspect the retained comment and biography history when needed.

- {data.tab === 'comments' ? 'VISIBLE COMMENTS' : 'VISIBLE BIOS'} - {data.tab === 'comments' ? data.summary.comments.visible : data.summary.bios.visible} - PUBLIC RECORDS + {data.tab === 'queue' ? 'PENDING REVIEW' : data.tab === 'comments' ? 'VISIBLE COMMENTS' : 'VISIBLE BIOS'} + {data.tab === 'queue' ? data.summary.queue.total : data.tab === 'comments' ? data.summary.comments.visible : data.summary.bios.visible} + {data.tab === 'queue' ? 'EXCEPTIONS ONLY' : 'PUBLIC RECORDS'}