From 22563e58a28132a288ee8e23309c3aa2d3614d83 Mon Sep 17 00:00:00 2001 From: Bonobo791 Date: Fri, 31 Jul 2026 07:18:59 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20retune=20AI=20thresholds=20=E2=80=94=20?= =?UTF-8?q?approve=20=E2=89=A40.50,=20queue=200.51=E2=80=930.75,=20reject?= =?UTF-8?q?=200.76=E2=80=930.94,=20ban=20=E2=89=A50.95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DESIGN.md | 2 +- EXECUTION_PLAN_YouTube_Comment_Moderator.md | 25 ++++++++++--------- PRODUCT.md | 8 +++--- src/lib/server/pipeline.test.ts | 19 ++++++++------ src/lib/server/pipeline.ts | 16 ++++++------ .../(app)/channels/[id]/queue/+page.svelte | 2 +- src/routes/+page.svelte | 10 ++++---- 7 files changed, 43 insertions(+), 39 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 90bfc641..b221e81d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -291,7 +291,7 @@ violet-tinted Gel Border, dashed only for empty states. ### Lighting plot (signature) Landing rule/threshold lists render as a cue sheet: a top rule line, rows of -stencil cue-kind labels (KEYWORD / ≥ 0.85) against their action, separated by +stencil cue-kind labels (KEYWORD / ≥ 0.95) against their action, separated by 28%-alpha currentColor hairlines, numerals always tabular. This is how the world shows a list of consequences — reuse it for any landing enumeration. diff --git a/EXECUTION_PLAN_YouTube_Comment_Moderator.md b/EXECUTION_PLAN_YouTube_Comment_Moderator.md index ea7e8893..24d3f2c0 100644 --- a/EXECUTION_PLAN_YouTube_Comment_Moderator.md +++ b/EXECUTION_PLAN_YouTube_Comment_Moderator.md @@ -39,7 +39,7 @@ By the end, there is a working SvelteKit app `yt-mod` that: 1. Lets a YouTube channel owner connect their channel via Google OAuth (scope `youtube.force-ssl`, offline access). The AES-256-GCM-encrypted refresh token is stored in the DB. 2. Stores per-channel rules: keyword, regex (recheck-validated), or blocked-user, each mapping to `hold` | `reject` | `delete` | `ban`. -3. Exposes `GET /api/cron?secret=` which processes **one channel per run** (least-recently-run first) and **one page of ≤100 comments per run**, with a persisted checkpoint (continuation token + high-water mark) so bursts larger than one page are drained across runs **without skipping comments**. Per comment: rule match first, then OpenAI Moderation. Score ≥ 0.85 → auto-reject; 0.35–0.85 → human review queue; < 0.35 → approved. Rule hits execute their configured action. **Every enforcement action is recorded in the DB (`action_pending`) BEFORE the YouTube write, then confirmed after** — a crash mid-run is reconciled on the next run, never repeated blindly and never lost. +3. Exposes `GET /api/cron?secret=` which processes **one channel per run** (least-recently-run first) and **one page of ≤100 comments per run**, with a persisted checkpoint (continuation token + high-water mark) so bursts larger than one page are drained across runs **without skipping comments**. Per comment: rule match first, then OpenAI Moderation. Score ≥ 0.95 → auto-ban (reject + ban author); 0.76–0.94 → auto-reject; 0.51–0.75 → human review queue; ≤ 0.50 → approved. Rule hits execute their configured action. **Every enforcement action is recorded in the DB (`action_pending`) BEFORE the YouTube write, then confirmed after** — a crash mid-run is reconciled on the next run, never repeated blindly and never lost. 4. Has four pages — dashboard (`/`), rules editor, review queue (one-click approve/reject/delete/ban), audit log — with a finished visual design (brand "Moderaty", design tokens, styled components, empty/loading/error states, accessible markup). 5. Supports `DRY_RUN=true`: classifies and writes audit rows (action `dry-run`) but performs **no** YouTube writes and **no** DB state changes (no comment rows, no cursor/checkpoint movement) — previews are repeatable. 6. `npm run check`, `npm run build`, `npm run test` all exit 0. @@ -57,8 +57,8 @@ API facts (use exactly; do not look up alternatives): - Channel lookup: `GET /channels?part=snippet&mine=true` - OAuth: auth URL `https://accounts.google.com/o/oauth2/v2/auth`, token endpoint `https://oauth2.googleapis.com/token`. Auth URL params: `client_id`, `redirect_uri`, `response_type=code`, `scope=https://www.googleapis.com/auth/youtube.force-ssl`, `access_type=offline`, `prompt=consent`. - OpenAI Moderation: `POST https://api.openai.com/v1/moderations`, header `Authorization: Bearer $OPENAI_API_KEY`, body `{ "model": "omni-moderation-latest", "input": "" }`. Response: `results[0].category_scores` (category → float). -- Toxicity categories: `harassment`, `harassment/threatening`, `hate`, `hate/threatening`, `violence`, `violence/graphic`. Comment AI score = max of these six. Spam is NOT an AI category; users catch spam with their own rules. -- Thresholds (fixed): AUTO_REJECT = 0.85, QUEUE = 0.35. +- Toxicity categories: `harassment`, `harassment/threatening`, `hate`, `hate/threatening`, `illicit`, `illicit/violent`, `self-harm`, `self-harm/intent`, `self-harm/instructions`, `sexual`, `sexual/minors`, `violence`, `violence/graphic`. Comment AI score = max of these thirteen. Spam is NOT an AI category; users catch spam with their own rules. +- Thresholds (fixed): AUTO_BAN = 0.95, AUTO_REJECT = 0.76, QUEUE = 0.51. - **Every field in every external response is optional until validated.** YouTube omits `authorChannelId` for deleted accounts; OpenAI may return out-of-range values. Handle per the invariants — never abort a batch over one malformed item. - Top-level comments only (replies are a non-goal). @@ -238,7 +238,7 @@ export const comments = sqliteTable('comments', { pendingAction: text('pending_action'), // 'hold' | 'reject' | 'delete' | 'ban' while action_pending; else null decidedBy: text('decided_by').notNull(), // 'rule' | 'ai' | 'human' | 'none' matchedRuleId: integer('matched_rule_id'), - aiScore: text('ai_score'), // JSON of the six category scores, or null + aiScore: text('ai_score'), // JSON of the thirteen category scores, or null createdAt: text('created_at').notNull() }); @@ -606,8 +606,9 @@ import { refreshAccessToken, fetchCommentPage, setModerationStatus, deleteCommen import { scoreComment, ModerationError } from './moderation'; import { matchRule } from './rules'; -const AUTO_REJECT = 0.85; -const QUEUE = 0.35; +const AUTO_BAN = 0.95; +const AUTO_REJECT = 0.76; +const QUEUE = 0.51; type PendingAction = 'hold' | 'reject' | 'delete' | 'ban'; @@ -918,7 +919,7 @@ function okResponse(scores: Record) { } describe('scoreComment', () => { - it('M1: returns max of the six toxic categories', async () => { + it('M1: returns max of the thirteen toxic categories', async () => { vi.stubGlobal('fetch', vi.fn(async () => okResponse({ harassment: 0.4, 'harassment/threatening': 0.1, hate: 0.9, 'hate/threatening': 0.1, violence: 0.1, 'violence/graphic': 0.1 }))); const m = await scoreComment('x'); expect(m.score).toBe(0.9); @@ -928,16 +929,16 @@ describe('scoreComment', () => { }); ``` -Note: a missing key among the six is INVALID (throws), per the module code and I2. Required cases: +Note: a missing key among the thirteen is INVALID (throws), per the module code and I2. Required cases: | # | Mocked API behavior | Expected | |---|---|---| -| M1 | 200, six valid scores, max = hate 0.9 | `score === 0.9`, `scores.hate === 0.9` | -| M2 | 200, score exactly 0.85 | returns 0.85 (boundary belongs to caller, not this module) | +| M1 | 200, thirteen valid scores, max = hate 0.9 | `score === 0.9`, `scores.hate === 0.9` | +| M2 | 200, score exactly 0.95 | returns 0.95 (boundary belongs to caller, not this module) | | M3 | 200, one score = 1.7 | throws `ModerationError` | | M4 | 200, one score = -0.2 | throws `ModerationError` | | M5 | 200, one score = "high" (string) | throws `ModerationError` | -| M6 | 200, one of the six keys missing | throws `ModerationError` | +| M6 | 200, one of the thirteen keys missing | throws `ModerationError` | | M7 | 200, `results` missing | throws `ModerationError` | | M8 | 500 status | throws `ModerationError` | | M9 | 200, non-JSON body | throws `ModerationError` | @@ -1512,7 +1513,7 @@ export const actions = {

Review queue — {data.ch?.title}

-

Borderline comments (AI score 0.35–0.85, or AI unavailable). Your action is final.

+

Borderline comments (AI score 0.51–0.75, or AI unavailable). Your action is final.

{#each data.pending as c}
diff --git a/PRODUCT.md b/PRODUCT.md index 9273ccd8..f03ef98d 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -51,10 +51,10 @@ Category line: "Comment protection for YouTube creators." - Rule types: keyword, regex (RE2 syntax only — never `new RegExp` on user patterns), blocked-user; actions: hold, reject, delete, ban. -- AI scoring: OpenAI Moderation `omni-moderation-latest`; score = max of six - toxicity categories; fixed thresholds >0.85 auto-ban, 0.51–0.85 auto-delete, - 0.35–0.50 human queue, <0.35 approved. AI failure always routes to the human - queue — never auto-approve, never auto-reject. +- AI scoring: OpenAI Moderation `omni-moderation-latest`; score = max of the + thirteen toxicity categories; fixed thresholds ≥0.95 auto-ban, 0.76–0.94 + auto-reject, 0.51–0.75 human queue, ≤0.50 approved. AI failure always routes + to the human queue — never auto-approve, never auto-reject. - Enforcement durability: every action is recorded locally (`action_pending`) BEFORE any YouTube write and confirmed after; `DRY_RUN=true` previews change nothing durable (audit rows only). diff --git a/src/lib/server/pipeline.test.ts b/src/lib/server/pipeline.test.ts index f00c305b..f3a45b7f 100644 --- a/src/lib/server/pipeline.test.ts +++ b/src/lib/server/pipeline.test.ts @@ -229,12 +229,12 @@ afterEach(() => { }); test.each([ - { score: 0.34, status: 'approved', queued: 0, acted: 0, api: 'none', audit: 'approve' }, - { score: 0.35, status: 'pending', queued: 1, acted: 0, api: 'none', audit: 'queue' }, - { score: 0.5, status: 'pending', queued: 1, acted: 0, api: 'none', audit: 'queue' }, - { score: 0.51, status: 'deleted', queued: 0, acted: 1, api: 'delete', audit: 'delete' }, - { score: 0.85, status: 'deleted', queued: 0, acted: 1, api: 'delete', audit: 'delete' }, - { score: 0.86, status: 'rejected', queued: 0, acted: 1, api: 'ban', audit: 'ban' } + { score: 0.5, status: 'approved', queued: 0, acted: 0, api: 'none', audit: 'approve' }, + { score: 0.51, status: 'pending', queued: 1, acted: 0, api: 'none', audit: 'queue' }, + { score: 0.75, status: 'pending', queued: 1, acted: 0, api: 'none', audit: 'queue' }, + { score: 0.76, status: 'rejected', queued: 0, acted: 1, api: 'reject', audit: 'reject' }, + { score: 0.94, status: 'rejected', queued: 0, acted: 1, api: 'reject', audit: 'reject' }, + { score: 0.95, status: 'rejected', queued: 0, acted: 1, api: 'ban', audit: 'ban' } ])('categorizes score $score as $status', async ({ score, status, queued, acted, api, audit }) => { mocks.scoreComment.mockResolvedValue(moderation(score)); @@ -245,6 +245,9 @@ test.each([ if (api === 'delete') { expect(mocks.deleteComment).toHaveBeenCalledWith('comment', 'access-token', undefined); expect(mocks.setModerationStatus).not.toHaveBeenCalled(); + } else if (api === 'reject') { + expect(mocks.setModerationStatus).toHaveBeenCalledWith(['comment'], 'rejected', false, 'access-token', undefined); + expect(mocks.deleteComment).not.toHaveBeenCalled(); } else if (api === 'ban') { expect(mocks.setModerationStatus).toHaveBeenCalledWith(['comment'], 'rejected', true, 'access-token', undefined); expect(mocks.deleteComment).not.toHaveBeenCalled(); @@ -454,8 +457,8 @@ test('retries a dispatched ban while the comment is still public', async () => { }); test.each([ - { raw: 0.506, status: 'deleted', reason: 'ai score 0.51' }, - { raw: 0.504, status: 'pending', reason: 'ai score 0.50' } + { raw: 0.506, status: 'pending', reason: 'ai score 0.51' }, + { raw: 0.504, status: 'approved', reason: 'ai score 0.50' } ])('rounds the AI score to 2 decimals before deciding ($raw → $status)', async ({ raw, status, reason }) => { mocks.scoreComment.mockResolvedValue(moderation(raw)); diff --git a/src/lib/server/pipeline.ts b/src/lib/server/pipeline.ts index 3e401e88..5e68582d 100644 --- a/src/lib/server/pipeline.ts +++ b/src/lib/server/pipeline.ts @@ -34,9 +34,9 @@ import { type NewComment } from '$lib/server/youtube'; -const AUTO_BAN = 0.85; -const AUTO_DELETE = 0.51; -const QUEUE = 0.35; +const AUTO_BAN = 0.95; +const AUTO_REJECT = 0.76; +const QUEUE = 0.51; export interface RunChannelOptions { maxPages?: number; @@ -131,7 +131,7 @@ async function aiDecision(comment: NewComment, deadline?: number): Promise AUTO_BAN) { + if (score >= AUTO_BAN) { return { comment, status: 'rejected', @@ -143,16 +143,16 @@ async function aiDecision(comment: NewComment, deadline?: number): Promise= AUTO_DELETE) { + if (score >= AUTO_REJECT) { return { comment, - status: 'deleted', + status: 'rejected', decidedBy: 'ai', matchedRuleId: null, aiScore, - auditAction: 'delete', + auditAction: 'reject', reason, - youtubeAction: 'delete' + youtubeAction: 'reject' }; } if (score >= QUEUE) { diff --git a/src/routes/(app)/channels/[id]/queue/+page.svelte b/src/routes/(app)/channels/[id]/queue/+page.svelte index 893e8cd8..22d72eb5 100644 --- a/src/routes/(app)/channels/[id]/queue/+page.svelte +++ b/src/routes/(app)/channels/[id]/queue/+page.svelte @@ -41,7 +41,7 @@ Commercial licensing: contact@marketingprowess.simplelogin.com — see COMMERCIA

Review queue — {data.ch?.title}

-

Borderline comments (AI score 0.35–0.50) waiting for your decision.

+

Borderline comments (AI score 0.51–0.75) waiting for your decision.

{#if data.pending === undefined} diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 5995ca73..8959bbc0 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -190,13 +190,13 @@ Commercial licensing: contact@marketingprowess.simplelogin.com — see COMMERCIA

Then the AI scores what your rules miss.

Every comment your rules don’t catch is scored by OpenAI’s moderation model - across six toxicity categories. The highest score decides: + across thirteen toxicity categories. The highest score decides:

-

> 0.85 AUTHOR BANNED AUTOMATICALLY

-

0.51 – 0.85 DELETED AUTOMATICALLY

-

0.35 – 0.50 HELD FOR YOUR REVIEW

-

< 0.35 APPROVED

+

≥ 0.95 AUTHOR BANNED AUTOMATICALLY

+

0.76 – 0.94 REJECTED AUTOMATICALLY

+

0.51 – 0.75 HELD FOR YOUR REVIEW

+

≤ 0.50 APPROVED

If the AI can’t score a comment, it lands in your queue — never auto-approved,