feat: add automated community moderation
This commit is contained in:
JB
+166
-1
@@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS avatar_reviews (
|
||||
storage_key TEXT NOT NULL,
|
||||
byte_size INTEGER NOT NULL,
|
||||
status TEXT NOT NULL CHECK(status IN ('pending', 'approved', 'removed', 'withdrawn', 'superseded')),
|
||||
review_origin TEXT NOT NULL DEFAULT 'legacy' CHECK(review_origin IN ('automated', 'user_report', 'legacy')),
|
||||
submitted_at TEXT NOT NULL,
|
||||
reviewed_at TEXT,
|
||||
reviewed_by TEXT REFERENCES users(id),
|
||||
@@ -150,7 +151,7 @@ CREATE TABLE IF NOT EXISTS comments (
|
||||
author_id TEXT NOT NULL REFERENCES users(id),
|
||||
parent_id TEXT REFERENCES comments(id),
|
||||
body_markdown TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'visible' CHECK(status IN ('visible', 'removed')),
|
||||
status TEXT NOT NULL DEFAULT 'pending_review' CHECK(status IN ('pending_review', 'visible', 'removed')),
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
removed_at TEXT,
|
||||
@@ -159,6 +160,170 @@ CREATE TABLE IF NOT EXISTS comments (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS comments_thread_status_created_idx ON comments(thread_id, status, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS comments_author_status_created_idx ON comments(author_id, status, created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS community_moderation_settings (
|
||||
id TEXT PRIMARY KEY CHECK(id='primary'),
|
||||
automated_enabled INTEGER NOT NULL DEFAULT 0 CHECK(automated_enabled IN (0,1)),
|
||||
auto_publish_clean INTEGER NOT NULL DEFAULT 1 CHECK(auto_publish_clean IN (0,1)),
|
||||
hold_on_provider_error INTEGER NOT NULL DEFAULT 1 CHECK(hold_on_provider_error IN (0,1)),
|
||||
scan_comments INTEGER NOT NULL DEFAULT 1 CHECK(scan_comments IN (0,1)),
|
||||
scan_profiles INTEGER NOT NULL DEFAULT 1 CHECK(scan_profiles IN (0,1)),
|
||||
scan_guides INTEGER NOT NULL DEFAULT 1 CHECK(scan_guides IN (0,1)),
|
||||
scan_avatars INTEGER NOT NULL DEFAULT 1 CHECK(scan_avatars IN (0,1)),
|
||||
provider_model TEXT NOT NULL DEFAULT 'omni-moderation-latest' CHECK(length(trim(provider_model))>0),
|
||||
detect_pii INTEGER NOT NULL DEFAULT 1 CHECK(detect_pii IN (0,1)),
|
||||
max_links INTEGER NOT NULL DEFAULT 3 CHECK(max_links BETWEEN 0 AND 100),
|
||||
duplicate_window_seconds INTEGER NOT NULL DEFAULT 300 CHECK(duplicate_window_seconds BETWEEN 0 AND 86400),
|
||||
policy_version INTEGER NOT NULL DEFAULT 1 CHECK(policy_version>=1),
|
||||
updated_at TEXT NOT NULL,
|
||||
updated_by TEXT REFERENCES users(id)
|
||||
);
|
||||
INSERT OR IGNORE INTO community_moderation_settings(
|
||||
id,automated_enabled,auto_publish_clean,hold_on_provider_error,
|
||||
scan_comments,scan_profiles,scan_guides,scan_avatars,provider_model,detect_pii,
|
||||
max_links,duplicate_window_seconds,policy_version,updated_at
|
||||
) VALUES(
|
||||
'primary',0,1,1,1,1,1,1,'omni-moderation-latest',1,3,300,1,
|
||||
'2026-07-27T00:00:00.000Z'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS community_moderation_thresholds (
|
||||
category TEXT PRIMARY KEY CHECK(length(trim(category))>0),
|
||||
review_threshold REAL NOT NULL CHECK(review_threshold BETWEEN 0 AND 1),
|
||||
priority_threshold REAL NOT NULL CHECK(priority_threshold BETWEEN 0 AND 1 AND priority_threshold>=review_threshold),
|
||||
enabled INTEGER NOT NULL DEFAULT 1 CHECK(enabled IN (0,1)),
|
||||
updated_at TEXT NOT NULL,
|
||||
updated_by TEXT REFERENCES users(id)
|
||||
);
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('harassment',0.30,0.70,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('harassment/threatening',0.10,0.40,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('hate',0.20,0.60,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('hate/threatening',0.10,0.40,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('illicit',0.30,0.70,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('illicit/violent',0.10,0.40,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('self-harm',0.15,0.50,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('self-harm/intent',0.10,0.35,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('self-harm/instructions',0.10,0.35,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('sexual',0.25,0.65,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('sexual/minors',0.05,0.20,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('violence',0.35,0.75,1,'2026-07-27T00:00:00.000Z');
|
||||
INSERT OR IGNORE INTO community_moderation_thresholds(category,review_threshold,priority_threshold,enabled,updated_at)
|
||||
VALUES('violence/graphic',0.20,0.55,1,'2026-07-27T00:00:00.000Z');
|
||||
CREATE TABLE IF NOT EXISTS community_moderation_terms (
|
||||
id TEXT PRIMARY KEY,
|
||||
list_kind TEXT NOT NULL CHECK(list_kind IN ('allow','ban')),
|
||||
match_mode TEXT NOT NULL CHECK(match_mode IN ('word','phrase')),
|
||||
value TEXT NOT NULL CHECK(length(trim(value))>0),
|
||||
normalized_value TEXT NOT NULL CHECK(length(trim(normalized_value))>0),
|
||||
scopes_json TEXT NOT NULL DEFAULT '["comment","profile_bio","guide"]',
|
||||
enabled INTEGER NOT NULL DEFAULT 1 CHECK(enabled IN (0,1)),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
created_by TEXT NOT NULL REFERENCES users(id),
|
||||
updated_at TEXT NOT NULL,
|
||||
updated_by TEXT NOT NULL REFERENCES users(id),
|
||||
UNIQUE(list_kind,match_mode,normalized_value,scopes_json)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS community_moderation_terms_enabled_kind_idx
|
||||
ON community_moderation_terms(enabled,list_kind,normalized_value);
|
||||
CREATE TABLE IF NOT EXISTS moderated_content_revisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL CHECK(content_type IN ('profile_bio','guide')),
|
||||
content_id TEXT NOT NULL,
|
||||
submitted_by TEXT NOT NULL REFERENCES users(id),
|
||||
body_markdown TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL CHECK(length(content_hash)>0),
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(content_type,content_id,content_hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS moderated_content_revisions_content_created_idx
|
||||
ON moderated_content_revisions(content_type,content_id,created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS content_moderation_scans (
|
||||
id TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL CHECK(content_type IN ('comment','profile_bio','guide','avatar')),
|
||||
content_id TEXT NOT NULL,
|
||||
author_id TEXT NOT NULL REFERENCES users(id),
|
||||
content_hash TEXT NOT NULL CHECK(length(content_hash)>0),
|
||||
revision_id TEXT REFERENCES moderated_content_revisions(id),
|
||||
policy_version INTEGER NOT NULL CHECK(policy_version>=1),
|
||||
provider TEXT NOT NULL CHECK(length(trim(provider))>0),
|
||||
model TEXT NOT NULL CHECK(length(trim(model))>0),
|
||||
provider_request_id TEXT,
|
||||
status TEXT NOT NULL CHECK(status IN ('approved','review','priority_review','error')),
|
||||
risk_level TEXT NOT NULL CHECK(risk_level IN ('low','medium','high')),
|
||||
reason TEXT NOT NULL CHECK(length(trim(reason))>0),
|
||||
scores_json TEXT NOT NULL DEFAULT '{}',
|
||||
flags_json TEXT NOT NULL DEFAULT '[]',
|
||||
rule_matches_json TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
error_code TEXT,
|
||||
error_message TEXT,
|
||||
UNIQUE(content_type,content_id,content_hash,policy_version,provider,model),
|
||||
CHECK(
|
||||
(content_type IN ('comment','avatar') AND revision_id IS NULL)
|
||||
OR (content_type IN ('profile_bio','guide') AND revision_id IS NOT NULL)
|
||||
),
|
||||
CHECK(
|
||||
(status='error' AND error_code IS NOT NULL)
|
||||
OR (status<>'error' AND error_code IS NULL AND error_message IS NULL)
|
||||
)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS content_moderation_scans_pending_queue_idx
|
||||
ON content_moderation_scans(status,created_at)
|
||||
WHERE status IN ('review','priority_review');
|
||||
CREATE INDEX IF NOT EXISTS content_moderation_scans_content_idx
|
||||
ON content_moderation_scans(content_type,content_id,created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS content_moderation_reviews (
|
||||
id TEXT PRIMARY KEY,
|
||||
scan_id TEXT NOT NULL REFERENCES content_moderation_scans(id),
|
||||
reviewer_id TEXT NOT NULL REFERENCES users(id),
|
||||
decision TEXT NOT NULL CHECK(decision IN ('approved','rejected','superseded')),
|
||||
reason TEXT NOT NULL CHECK(length(trim(reason))>0),
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS content_moderation_reviews_scan_created_idx
|
||||
ON content_moderation_reviews(scan_id,created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS content_moderation_reviews_reviewer_created_idx
|
||||
ON content_moderation_reviews(reviewer_id,created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS user_content_reports (
|
||||
id TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL CHECK(content_type IN ('comment','profile_bio','guide','avatar')),
|
||||
content_id TEXT NOT NULL,
|
||||
reporter_id TEXT NOT NULL REFERENCES users(id),
|
||||
category TEXT NOT NULL CHECK(category IN (
|
||||
'spam','harassment','hate','sexual','violence','self_harm','privacy','other'
|
||||
)),
|
||||
details TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','resolved','dismissed')),
|
||||
created_at TEXT NOT NULL,
|
||||
reviewed_at TEXT,
|
||||
reviewed_by TEXT REFERENCES users(id),
|
||||
resolution_reason TEXT,
|
||||
CHECK(
|
||||
(status='pending' AND reviewed_at IS NULL AND reviewed_by IS NULL AND resolution_reason IS NULL)
|
||||
OR (status IN ('resolved','dismissed') AND reviewed_at IS NOT NULL AND reviewed_by IS NOT NULL
|
||||
AND resolution_reason IS NOT NULL AND length(trim(resolution_reason))>0)
|
||||
)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS user_content_reports_pending_queue_idx
|
||||
ON user_content_reports(created_at)
|
||||
WHERE status='pending';
|
||||
CREATE INDEX IF NOT EXISTS user_content_reports_content_idx
|
||||
ON user_content_reports(content_type,content_id,created_at DESC);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS user_content_reports_one_pending_idx
|
||||
ON user_content_reports(reporter_id,content_type,content_id)
|
||||
WHERE status='pending';
|
||||
CREATE TABLE IF NOT EXISTS favorites (user_id TEXT NOT NULL REFERENCES users(id), build_id TEXT NOT NULL REFERENCES builds(id) ON DELETE CASCADE, created_at TEXT NOT NULL, PRIMARY KEY(user_id, build_id));
|
||||
CREATE TABLE IF NOT EXISTS equipment_types (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, category TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS stat_definitions (id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, value_kind TEXT NOT NULL, min_value REAL, max_value REAL, unit TEXT, active INTEGER NOT NULL DEFAULT 1);
|
||||
|
||||
Reference in New Issue
Block a user