diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84645f6..8c84731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ jobs: - name: Run edge function tests run: deno test --allow-all supabase/functions/ + - name: Check devotional request handler + run: deno check supabase/functions/daily-devotional/index.ts + parser-tests: name: Parser unit tests (Python) runs-on: ubuntu-latest diff --git a/.github/workflows/deploy-supabase-functions.yml b/.github/workflows/deploy-supabase-functions.yml index 3f56bcf..8b61ae2 100644 --- a/.github/workflows/deploy-supabase-functions.yml +++ b/.github/workflows/deploy-supabase-functions.yml @@ -5,11 +5,24 @@ on: branches: [master] paths: - "supabase/functions/**" + - "supabase/migrations/**" - ".github/workflows/deploy-supabase-functions.yml" + - ".github/workflows/deploy-supabase-migrations.yml" workflow_dispatch: +concurrency: + group: supabase-production + cancel-in-progress: false + jobs: + migrations: + uses: ./.github/workflows/deploy-supabase-migrations.yml + with: + serialized_by_caller: true + secrets: inherit + deploy: + needs: migrations runs-on: ubuntu-latest env: diff --git a/.github/workflows/deploy-supabase-migrations.yml b/.github/workflows/deploy-supabase-migrations.yml index c1fe643..efba86b 100644 --- a/.github/workflows/deploy-supabase-migrations.yml +++ b/.github/workflows/deploy-supabase-migrations.yml @@ -1,13 +1,19 @@ name: Deploy Supabase Database Migrations on: - push: - branches: [master] - paths: - - "supabase/migrations/**" - - ".github/workflows/deploy-supabase-migrations.yml" + workflow_call: + inputs: + serialized_by_caller: + type: boolean + default: false workflow_dispatch: +# Reusable calls already run inside the parent's production lock. Manual +# migration runs acquire that same lock; re-acquiring it in a child deadlocks. +concurrency: + group: ${{ inputs.serialized_by_caller && format('supabase-migrations-{0}', github.run_id) || 'supabase-production' }} + cancel-in-progress: false + jobs: deploy-migrations: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 172d838..b893663 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ functions functions-background \ ngrok-up ngrok-down ngrok-background \ functions-deploy test_daily_devotional test_slowness fresh \ - test test-ios test-android test-parsers \ + test test-ios test-android test-parsers test-edge eval-devotional-judge \ archive upload release \ pull-listings dry-listings push-listings push-listing create-version \ pull-screenshots dry-screenshots push-screenshots \ @@ -31,6 +31,8 @@ help: @echo " make test-ios – iOS unit tests (swiftbibleTests, simulator)" @echo " make test-android – Android unit tests (gradle)" @echo " make test-parsers – Python parser unit tests (stdlib unittest)" + @echo " make test-edge – Edge function tests (Deno)" + @echo " make eval-devotional-judge – Live devotional-judge evals (needs OPENAI_API_KEY)" @echo "" @echo "Deploy:" @echo " make functions-deploy – Deploy all functions to remote project" @@ -154,6 +156,15 @@ test-android: test-parsers: python3 -m unittest discover -s python_parser +test-edge: + deno test --allow-all supabase/functions/ + +# Live evals for the daily-devotional quality judge. Calls OpenAI (costs a few +# cents); needs OPENAI_API_KEY. TRIALS=3 runs each labelled case 3 times. +eval-devotional-judge: + RUN_JUDGE_EVALS=1 JUDGE_EVAL_TRIALS=$${TRIALS:-1} \ + deno test --allow-all supabase/functions/daily-devotional/judge_eval_test.ts + test: test-ios test-android test-parsers @echo "✅ All unit tests passed" diff --git a/supabase/functions/_shared/devotional-examples.ts b/supabase/functions/_shared/devotional-examples.ts new file mode 100644 index 0000000..3767bff --- /dev/null +++ b/supabase/functions/_shared/devotional-examples.ts @@ -0,0 +1,66 @@ +// The writer and live evaluation use the exact same worked example. +// Keep imagined details out of biblical scenes; put the observable opening +// in the reader's world, then distinguish the verse from its context. +export const EMPATHY_EXAMPLE = `# May 15 — Mark 1:35: Before the next request + +**Where does prayer fit when the work is still waiting?** + +> *"And in the morning, rising up a great while before day, he went out, and departed into a solitary place, and there prayed."* +> **Mark 1:35** + +## Before opening the messages + +A phone lies beside a half-filled mug. Even with the screen dark, it can hold your attention. If you have ever reached for it before you knew what you wanted to read, you know how quickly a morning can fill with other people's requests. + +Leaving it there for a few minutes may feel surprisingly difficult. Someone could need you. + +## The work was real + +Mark places this prayer after an evening of healing. The previous verses describe people bringing the sick to Jesus and the city gathered at the door. Then, a great while before day, Jesus goes out to a solitary place and prays. + +The passage gives us his action without reporting his prayer's words or his private reasons for choosing that hour. The Greek word translated "solitary" is also used for wilderness or an uninhabited place. Here, Mark names somewhere away from the gathering crowd. + +The requests have not ended. In the following verses, Simon and the others find Jesus and tell him that everyone is looking for him. Prayer takes place within a ministry full of actual needs. Mark does not describe a quiet season with nothing pressing to do. + +## While things remain unfinished + +If you are waiting for every obligation to be settled before you pray, this scene gives you something to reconsider. Jesus has people seeking him, and he prays. We need not know everything about his morning to notice that. + +You may have a few minutes before opening your messages, or a pause later in the day. Bring the unfinished work to the Father in that time. It may still be waiting when you finish praying, and someone may still be asking for you. + +## A prayer + +*Father, help me bring You the work I have not finished and the people whose needs I cannot settle. Give me the willingness to pray while things remain unresolved, and the patience to return to those people with care. Amen.*`; + +export const NARRATIVE_EXAMPLE = `# October 12 — Luke 19:5: A name above the crowd + +**The crowd calls him a sinner; Jesus calls him Zacchaeus.** + +> *"And when Jesus came to the place, he looked up, and saw him, and said unto him, Zacchaeus, make haste, and come down; for to day I must abide at thy house."* +> **Luke 19:5** + +## Up in the sycamore + +Zacchaeus is up in a tree. The preceding verses tell us how he got there: a short man, unable to see Jesus through the crowd, runs ahead and climbs a sycamore along the route. He wants to see who Jesus is. + +Below him, Jesus comes to the place and looks up. He sees the man in the branches and speaks his name. Then comes the command to hurry down, with a reason attached: today Jesus must stay at his house. + +The man who ran ahead to get a look has been asked to receive a guest. + +## The name and the complaint + +In the next verse, Zacchaeus makes haste, comes down and receives him joyfully. His movement answers the words Jesus spoke from below. The tree has served its purpose; the visit is now the concern. + +Around that welcome, another response takes shape. Luke records the onlookers murmuring that Jesus has gone to be the guest of a sinner. Jesus has addressed Zacchaeus by name. The complaint identifies him by the category that makes the visit objectionable. + +The account does not deny that sin matters. A little later, Zacchaeus speaks about giving to the poor and repaying anyone he has defrauded. But the crowd's objection is to Jesus going into his house at all. Luke lets us hear that objection beside the welcome. + +## Whose response resembles yours? + +If there is someone you find easy to dismiss, that contrast deserves time. Knowing something true about a person's conduct can become a reason to resent anyone showing them kindness. In this scene, Jesus goes to the house the crowd thinks he should avoid. + +Consider one ordinary act of welcome you have been withholding. Offering it will not settle every question about the person. The murmuring Luke records reminds us how readily a welcome can draw objections, including our own. + +## A prayer + +*Father, keep us from resenting the mercy Your Son shows. Help us see the people we have reduced to a reputation, and teach us to offer welcome without pretending that sin is harmless. Give us patience for the questions we cannot settle at the door. Amen.*`; diff --git a/supabase/functions/_shared/devotional-judge.ts b/supabase/functions/_shared/devotional-judge.ts new file mode 100644 index 0000000..722a809 --- /dev/null +++ b/supabase/functions/_shared/devotional-judge.ts @@ -0,0 +1,407 @@ +// Quality gate for the daily devotional. +// +// After a devotional is generated and before it is saved, a second model call +// grades it against a fixed rubric. Below the threshold (or on any hard fail) +// the failed judgment is recorded in daily_devotional_rejections; +// the function regenerates on the same track, type and testament from a +// different verse. +// +// Lives in _shared for the same reason verse-selection.ts does: +// daily-devotional/index.ts calls Deno.serve at top level and cannot be +// imported by a test. Everything with logic (prompt, parsing, pass/fail, the +// retry loop) is here and exercised by daily-devotional/judge_test.ts; the +// live-model evals are daily-devotional/judge_eval_test.ts. +// +// The model id is NOT set here. JUDGE_MODEL lives beside DEVOTIONAL_MODEL in +// daily-devotional/index.ts (one source of truth for every model) and is +// passed in. +// +// Pass/fail is decided in code from the per-criterion scores, never from a +// verdict the model writes. A model asked "pass or fail?" drifts toward pass; +// asked for scores against anchored descriptions, it is far steadier, and the +// threshold stays a number we can tune and test. + +import { PROMPT_THEOLOGY_GUARDRAILS } from "./devotional-voice.ts"; + +// Bump when the rubric or prompt changes, so judge results stored on rows can +// be compared like with like (mirrors prompt_version on the devotional). +export const JUDGE_PROMPT_VERSION = "judge-v1"; + +// Overall score is the weighted mean of the criteria, scaled to 0–10. +// 7.0 means "solidly good on average, nothing weak enough to drag it down". +// Tune against judge_eval_test.ts, not by feel. +export const JUDGE_PASS_THRESHOLD = 7.0; + +// Any single criterion at or below this fails the draft regardless of the +// average. A devotional that is beautifully written about a verse that cannot +// carry it is still a bad devotional. +export const JUDGE_CRITERION_FLOOR = 2; + +// Total generations per day, including the first. Each attempt is one +// generation (~20–60s) plus one judge call, and the Edge Function has a wall +// clock limit, so keep this small. +export const JUDGE_MAX_ATTEMPTS = 3; + +// Leave 30 seconds below Supabase's 150-second request timeout for audit, +// persistence and error reporting. All model calls share this deadline. +export const GENERATION_BUDGET_MS = 120_000; + +export const JUDGE_CRITERIA = [ + "verse_weight", + "faithfulness", + "substance", + "track_fit", + "craft", +] as const; +export type JudgeCriterion = typeof JUDGE_CRITERIA[number]; + +// faithfulness and verse_weight carry the most: a misread text or a verse +// stretched past what it says is the failure readers notice and distrust. +export const JUDGE_WEIGHTS: Record = { + verse_weight: 1.5, + faithfulness: 1.5, + substance: 1.0, + track_fit: 1.0, + craft: 0.75, +}; + +export const JUDGE_HARD_FAILS = [ + "misquoted_scripture", + "factual_error", + "theological_error", + "verse_cannot_carry_devotional", + "broken_format", + "inappropriate_content", +] as const; +export type JudgeHardFail = typeof JUDGE_HARD_FAILS[number]; + +// What each track is FOR, so the judge grades it on its own terms. Without +// this the lament track (which deliberately does not resolve) is marked down +// for not resolving, and the matt track for being terse. +export const TRACK_INTENT: Record = { + empathy: + "Four beats: an observable opening scene, what the verse actually says (one short Hebrew/Greek note at most), bringing the two together honestly, then a short prayer. Quiet pastoral warmth.", + technical: + "Contextual, historical, cultural and linguistic study of the verse, then reflection questions and an italic closing meditation. Heavier on biblical background than modern application.", + narrative: + "Retells the scene in the text vividly without inventing dialogue, motives or events scripture does not supply, then lands on the reader.", + practical: + "Draws one concrete action from the verse, names how it typically goes wrong, and what the text says to do then.", + matt: + "Terse, dry, precise on the text. Concrete domestic opening, a close reading of which words carry the weight, then a few short imperatives. Spare by design; do not mark it down for brevity.", + josh: + "Teacher's register. Opens on a real question, sets the text in its original world with honest uncertainty, contrasts the ancient and modern sense of a key word, hands the question back.", + lament: + "Deliberately does NOT resolve. Names a real loss in the passage plainly, stays inside the text's grief without explaining it away, ends with the sufferer still speaking to God. Do not mark it down for lacking comfort or a lesson. DO mark it down if the passage contains no real grief, loss or protest and the devotional has to import one.", + question: + "Opens on one real, possibly uncomfortable question, states the strongest objection fairly, separates what the text states from what it implies, ends with the question sharpened.", + character: + "Follows one biblical person across an arc (not a single scene), shows change over time and what God did with their failure, and keeps them a person rather than a moral type.", +}; + +export interface JudgeVerse { + book: string; + chapter: number; + verse: number; + text: string; +} + +export interface JudgeInput { + message: string; + track: string; + verses: JudgeVerse[]; + holidayName?: string | null; +} + +export interface JudgeResult { + version: string; + scores: Record; + overall: number; + threshold: number; + hard_fails: JudgeHardFail[]; + passed: boolean; + // Short, specific notes from the judge. Stored for review; never shown. + reasons: string[]; +} + +export function buildJudgePrompt(input: JudgeInput): string { + const verseLines = input.verses + .map((v) => `${v.book} ${v.chapter}:${v.verse} — "${v.text}"`) + .join("\n"); + const intent = TRACK_INTENT[input.track] ?? + "A daily devotional: faithful to the text, specific, and ending in prayer."; + const holiday = input.holidayName + ? `\nHOLIDAY\nWritten for ${input.holidayName}; the holiday framing is expected.\n` + : ""; + + return `You are the editor deciding whether today's devotional is good enough to publish in SwiftBible, a Bible app read daily by a wide range of people. Grade it strictly against the rubric. You are not the author; do not be generous. + +ANCHOR VERSE(S) — KJV, exact text +${verseLines} + +TRACK: ${input.track} +What this track is for: ${intent} +${holiday} +The house theology the author was given: +${PROMPT_THEOLOGY_GUARDRAILS} + +RUBRIC — score each criterion 1 to 5 using these anchors. + +verse_weight — can this passage carry a devotional, and does the piece draw out what is actually there? + 5: rich passage, and the devotional's main point plainly comes from its words. + 3: the passage is thin, or the devotional leans on neighbouring verses for most of its point. + 1: a census line, genealogy, list of names, measurement or itinerary stretched over a theme it does not contain; the devotional mostly talks about something the verse does not say. + +faithfulness — accuracy to scripture and to the guardrails above. + 5: every quotation and claim about the text, its context and its people is correct; no invented scenes or motives. + 3: a loose paraphrase or an unhedged speculative claim. + 1: misquotes scripture, gets a fact about the text wrong, or contradicts the guardrails. + +substance — does the reader come away with something specific and true? + 5: a concrete insight a thoughtful reader would not have had on their own. + 3: correct but generic; could be written about many verses. + 1: vague, padded, or so short it says almost nothing. + +track_fit — does it do what its track is for (see above), on the track's own terms? + 5: clearly that track, done well. 3: recognisable but half-done. 1: wrong mode entirely. + +craft — prose and format. + 5: clean, varied, human; follows the markdown skeleton (title line, bolded subtitle, verse blockquote, sections, italic prayer ending in Amen). + 3: noticeable AI tells (em dashes in prose, "not X but Y" pivots, stacked threes, throat-clearing) or a thin section. + 1: broken structure, missing the prayer, or reads as machine output. + +HARD FAILS — list any that apply, otherwise an empty list: + misquoted_scripture, factual_error, theological_error, verse_cannot_carry_devotional, broken_format, inappropriate_content +Only flag one when you can point to the exact sentence. + +DEVOTIONAL TO GRADE +<<< +${input.message} +>>> + +Return ONLY a JSON object, no commentary: +{ + "scores": { "verse_weight": 1-5, "faithfulness": 1-5, "substance": 1-5, "track_fit": 1-5, "craft": 1-5 }, + "hard_fails": [], + "reasons": ["one short, specific sentence per weakness, quoting the devotional where useful"] +}`; +} + +function clampScore(raw: unknown): number | null { + const n = typeof raw === "string" ? Number(raw) : raw; + if (typeof n !== "number" || !Number.isFinite(n)) return null; + return Math.min(5, Math.max(1, Math.round(n))); +} + +export function overallScore(scores: Record): number { + let weighted = 0; + let total = 0; + for (const c of JUDGE_CRITERIA) { + weighted += scores[c] * JUDGE_WEIGHTS[c]; + total += JUDGE_WEIGHTS[c]; + } + // 1–5 mean → 0–10, so 1 maps to 0 and 5 to 10. + const mean = weighted / total; + return Math.round(((mean - 1) / 4) * 100) / 10; +} + +// Turns the judge model's JSON into a decision. Throws on anything it cannot +// score — the caller treats a throw as "judge unavailable", not as a fail. +export function parseJudgeResponse( + content: string, + threshold = JUDGE_PASS_THRESHOLD, +): JudgeResult { + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch { + // Tolerate a fenced or prefixed object. + const m = content.match(/\{[\s\S]*\}/); + if (!m) throw new Error(`judge returned no JSON: ${content.slice(0, 200)}`); + parsed = JSON.parse(m[0]); + } + const obj = parsed as { + scores?: Record; + hard_fails?: unknown; + reasons?: unknown; + }; + if (!obj || typeof obj !== "object" || !obj.scores) { + throw new Error(`judge response missing scores: ${content.slice(0, 200)}`); + } + + const scores = {} as Record; + for (const c of JUDGE_CRITERIA) { + const s = clampScore(obj.scores[c]); + if (s === null) throw new Error(`judge response missing score for ${c}`); + scores[c] = s; + } + + const hard_fails = Array.isArray(obj.hard_fails) + ? [...new Set( + obj.hard_fails.filter((h): h is JudgeHardFail => + (JUDGE_HARD_FAILS as readonly string[]).includes(h as string) + ), + )] + : []; + const reasons = Array.isArray(obj.reasons) + ? obj.reasons.filter((r): r is string => typeof r === "string").slice(0, 10) + : []; + + const overall = overallScore(scores); + const belowFloor = JUDGE_CRITERIA.some((c) => scores[c] <= JUDGE_CRITERION_FLOOR); + const passed = overall >= threshold && hard_fails.length === 0 && !belowFloor; + + return { + version: JUDGE_PROMPT_VERSION, + scores, + overall, + threshold, + hard_fails, + passed, + reasons, + }; +} + +// One OpenAI call. Returns the raw response body too, so the caller can cost +// it with the same extractUsage() it uses for generation. +export async function callJudgeModel(opts: { + apiKey: string; + model: string; + prompt: string; + fetchFn?: typeof fetch; + url?: string; + signal?: AbortSignal; +}): Promise<{ content: string; data: unknown }> { + const doFetch = opts.fetchFn ?? fetch; + const response = await doFetch( + opts.url ?? "https://api.openai.com/v1/chat/completions", + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${opts.apiKey}`, + }, + body: JSON.stringify({ + model: opts.model, + messages: [ + { + role: "system", + content: + "You are a strict editor of Christian devotionals. Return only valid JSON.", + }, + { role: "user", content: opts.prompt }, + ], + response_format: { type: "json_object" }, + max_completion_tokens: 1500, + }), + signal: AbortSignal.any([AbortSignal.timeout(60000), ...(opts.signal ? [opts.signal] : [])]), + }, + ); + if (!response.ok) { + throw new Error(`judge call failed (${response.status}): ${await response.text()}`); + } + const data = await response.json(); + const content = data?.choices?.[0]?.message?.content; + if (typeof content !== "string" || content.trim().length === 0) { + throw new Error(`judge returned empty content: ${JSON.stringify(data).slice(0, 300)}`); + } + return { content, data }; +} + +// ─── Retry loop ───────────────────────────────────────────────────── + +export interface JudgedAttempt { + attempt: number; // 1-based + candidate: C; + generation: G; + // null when the judge itself failed (network, bad JSON). Fail open: an + // outage of the grader must not stop the day's devotional. + judge: JudgeResult | null; + judgeError?: string; +} + +export interface JudgedOutcome { + accepted: JudgedAttempt | null; + // Every attempt that will NOT be published. Record these; never show them. + rejected: JudgedAttempt[]; + // True when every attempt failed and `accepted` is the best of the failures. + exhausted: boolean; + retryError?: string; +} + +// Generates, judges, and on failure moves to a fresh candidate (a different +// verse on the same track) until one passes or attempts run out. Generic over +// the candidate and generation shapes so the test can drive it with stubs. +// +// On exhaustion or a retry error, use the best completed draft only if it has +// no hard failures and no criterion below the floor. A low overall score may +// be tolerable; known errors are not. With no usable draft return null so the +// handler preserves any existing row. Audit failed judgments immediately, +// including a draft subsequently used as a soft-score fallback. +export async function generateWithJudge(opts: { + first: C; + next: (tried: C[]) => Promise; + generate: (candidate: C) => Promise; + judge: (candidate: C, generation: G) => Promise; + maxAttempts?: number; + signal?: AbortSignal; + onFailedAttempt?: (attempt: JudgedAttempt) => Promise; + log?: (msg: string) => void; +}): Promise> { + const max = Math.max(1, opts.maxAttempts ?? JUDGE_MAX_ATTEMPTS); + const log = opts.log ?? console.log; + const attempts: JudgedAttempt[] = []; + const tried: C[] = []; + let candidate = opts.first; + let retryError: string | undefined; + + for (let n = 1; n <= max; n++) { + let generation: G; + try { + opts.signal?.throwIfAborted(); + if (n > 1) candidate = await opts.next(tried); + tried.push(candidate); + generation = await opts.generate(candidate); + } catch (err) { + if (attempts.length === 0) throw err; + retryError = err instanceof Error ? err.message : String(err); + log(`judge attempt ${n}: retry stopped (${retryError})`); + break; + } + let judge: JudgeResult | null = null; + let judgeError: string | undefined; + try { + judge = await opts.judge(candidate, generation); + } catch (err) { + judgeError = err instanceof Error ? err.message : String(err); + log(`judge attempt ${n}: judge unavailable, accepting unjudged (${judgeError})`); + } + + const attempt: JudgedAttempt = { attempt: n, candidate, generation, judge, judgeError }; + attempts.push(attempt); + + if (judge === null || judge.passed) { + if (judge) log(`judge attempt ${n}: PASS ${judge.overall}/${judge.threshold}`); + return { accepted: attempt, rejected: attempts.slice(0, -1), exhausted: false }; + } + log( + `judge attempt ${n}: FAIL ${judge.overall}/${judge.threshold}` + + (judge.hard_fails.length ? ` hard_fails=${judge.hard_fails.join(",")}` : ""), + ); + // Persist before starting another slow external request. Audit errors do + // not discard the completed draft or stop the remaining quality checks. + try { + await opts.onFailedAttempt?.(attempt); + } catch (err) { + log(`judge attempt ${n}: audit failed (${String(err)})`); + } + } + + const usable = attempts.filter((a) => a.judge && a.judge.hard_fails.length === 0 && + JUDGE_CRITERIA.every((c) => a.judge!.scores[c] > JUDGE_CRITERION_FLOOR)); + const best = usable.reduce | null>( + (a, b) => !a || b.judge!.overall > a.judge!.overall ? b : a, null, + ); + log(best ? `judge: using soft-score fallback (attempt ${best.attempt}, ${best.judge!.overall})` : "judge: no usable draft; preserving existing content"); + return { accepted: best, rejected: attempts.filter((a) => a !== best), exhausted: true, retryError }; +} diff --git a/supabase/functions/_shared/sentry.ts b/supabase/functions/_shared/sentry.ts index 3b56e19..5d18165 100644 --- a/supabase/functions/_shared/sentry.ts +++ b/supabase/functions/_shared/sentry.ts @@ -89,6 +89,7 @@ export async function captureException( "X-Sentry-Auth": `Sentry sentry_version=7, sentry_key=${sentryPublicKey}`, }, body: envelope, + signal: AbortSignal.timeout(5000), }); if (!resp.ok) { console.warn("[sentry] ingest responded", resp.status); diff --git a/supabase/functions/_shared/verse-selection.ts b/supabase/functions/_shared/verse-selection.ts index b99df7c..764507e 100644 --- a/supabase/functions/_shared/verse-selection.ts +++ b/supabase/functions/_shared/verse-selection.ts @@ -35,11 +35,58 @@ export interface VerseBook { }>; } +export function verseKey(v: { book: string; chapter: number; verse: number }): string { + return `${v.book} ${v.chapter}:${v.verse}`; +} + // Paragraphs shorter than this are fragments; longer ones are genealogies and -// census lists. Applied once when the index is built, not per attempt. +// census lists (short ones are caught by UNSUITABLE_PATTERNS below). Applied +// once when the index is built, not per attempt. export const MIN_PARAGRAPH_CHARS = 20; export const MAX_PARAGRAPH_CHARS = 400; +// Passage shapes that cannot carry a devotional however well it is written. +// Checked before generation so a census line never costs a generation plus a +// judge round (the 2026-10-11 Numbers 26:58 lament was one). The judge after +// generation stays the backstop for everything a pattern cannot see. +// +// Deliberately narrow. Each rule matches one structural shape, and was +// measured against the whole pool: together they drop ~177 of ~24,000 +// eligible paragraphs, all but one OT (Acts 7:8, a begat recap). A broader +// "too many proper names" rule was tried and rejected: it took the cross's +// title (John 19:19), Mary Magdalene at the tomb (Mark 15:47) and the calling +// of the twelve (Matthew 10:2). A missed bad verse costs one retry; a blocked +// good one costs nothing visible only while the rules stay this narrow. +// +// Add a rule when daily_devotional_rejections shows a recurring shape the +// judge fails for verse_weight, and add the case to verse_selection_test.ts. +export const UNSUITABLE_PATTERNS: Array<{ name: string; pattern: RegExp; min: number }> = [ + // "And Attai begat Nathan, and Nathan begat Zabad…" + { name: "begat-chain", pattern: /\bbegat\b/g, min: 2 }, + // "And they departed from Rithmah, and pitched at Rimmonparez." + { + name: "itinerary", + pattern: /\b(?:departed|removed|journeyed|went) from [A-Z][\w-]*,? and (?:pitched|encamped)\b/g, + min: 1, + }, + // "the family of the Libnites, the family of the Hebronites…" + { name: "clan-list", pattern: /\bthe family of the \w+ites\b/g, min: 2 }, + // Tabernacle/temple specifications. No "talents": that word is the parable. + { + name: "measurements", + pattern: + /\b(?:cubits?|sockets?|boards?|bars|basons|breadth|shekels?|tenons|loops|taches|ouches|chapiters|pomegranates)\b/g, + min: 3, + }, +]; + +export function unsuitableReason(text: string): string | null { + for (const rule of UNSUITABLE_PATTERNS) { + if ((text.match(rule.pattern)?.length ?? 0) >= rule.min) return rule.name; + } + return null; +} + function stripJesusTags(text: string): string { return text.replace(/<\/?JESUS>/g, "").trim(); } @@ -61,11 +108,13 @@ export function eligibleParagraphs( for (const paragraph of chapter.paragraphs) { if (paragraph.text.length > MAX_PARAGRAPH_CHARS) continue; if (paragraph.text.trim().length < MIN_PARAGRAPH_CHARS) continue; + const text = stripJesusTags(paragraph.text); + if (unsuitableReason(text)) continue; out.push({ book: book.name, chapter: chapter.number, verse: paragraph.startingVerse, - text: stripJesusTags(paragraph.text), + text, testament, }); } @@ -78,12 +127,15 @@ export function eligibleParagraphs( export function selectRandomVerse( books: VerseBook[], testament: "old" | "new", - fallbackBookName: string + fallbackBookName: string, + excluded: ReadonlySet = new Set(), ): SelectedVerse { - const pool = eligibleParagraphs(books, testament); + const eligible = eligibleParagraphs(books, testament); + const pool = excluded.size ? eligible.filter((v) => !excluded.has(verseKey(v))) : eligible; if (pool.length > 0) { return pool[Math.floor(Math.random() * pool.length)]; } + if (eligible.length) throw new Error("verse selection: no unused eligible passages remain"); // Unreachable unless the bundled text failed to load. const fallbackBook = books.find((b) => b.name === fallbackBookName); diff --git a/supabase/functions/daily-devotional/README.md b/supabase/functions/daily-devotional/README.md index bff49a6..148bd63 100644 --- a/supabase/functions/daily-devotional/README.md +++ b/supabase/functions/daily-devotional/README.md @@ -1,209 +1,106 @@ # Daily Devotional Edge Function -Generates AI-powered daily devotionals with verse-accurate KJV text. Runs as a Supabase Edge Function triggered daily. - -## Devotional Types - -There are two devotional types that rotate on a 3-day cycle: - -| Day | Type | Testament | Description | -|-----|------|-----------|-------------| -| 1 | Single | Old | One OT verse, full devotional | -| 2 | Single | New | One NT verse, full devotional | -| 3 | Multi | Random | 2 thematically connected verses | -| 4 | Single | Old | Cycle repeats... | - -**Holidays override the cycle** and always produce a multi-verse devotional themed to the holiday. - ---- - -## Flow: Single Verse (Old Testament) - -``` -determineDevotionalType() - ├── checks yesterday's DB entry - └── yesterday was "multi" → today is "single", testament: "old" - -selectRandomVerse("old") - ├── picks random book from OT_BOOKS (Genesis–Malachi) - ├── picks random chapter → random paragraph - ├── filters: skip if > 400 chars (genealogies) or < 20 chars - ├── retries up to 10 times - └── fallback: Psalms (always devotional-worthy) - -createPrompt(verse, date, null) - └── single-verse prompt template with verse text - -generateDevotional(prompt) [gpt-5.4] - └── returns markdown devotional - -saveDevotional(message, date, "old", "single", [verse]) - └── upserts to "Daily Devotional" table -``` - -## Flow: Single Verse (New Testament) - -``` -determineDevotionalType() - ├── checks yesterday's DB entry - └── yesterday was "single" + "old" → today is "single", testament: "new" - -selectRandomVerse("new") - ├── picks random book from NT_BOOKS (Matthew–Revelation) - ├── same filtering as OT (skip long/short paragraphs) - └── fallback: John - -createPrompt(verse, date, null) - └── single-verse prompt template - -generateDevotional(prompt) [gpt-5.4] - -saveDevotional(message, date, "new", "single", [verse]) -``` - -## Flow: Multi-Verse (Non-Holiday) - -``` -determineDevotionalType() - ├── checks yesterday's DB entry - └── yesterday was "single" + "new" → today is "multi" - -verseCount = random(2 or 3) - -selectMultiVerses(verseCount, null) [gpt-5-mini] - ├── prompt: "pick N thematically connected KJV verses" - ├── response_format: json_object - ├── returns: [{ book, chapter, verse }, ...] - └── fast + cheap (~300 tokens max) - -for each verse ref: - lookupVerseText(book, chapter, verse) [bible.json] - ├── finds exact paragraph text - ├── strips tags - └── fallback: "[Book Chapter:Verse]" if not found - - determine testament: - └── OT_BOOKS.includes(book) ? "old" : "new" - -createMultiVersePrompt(resolvedVerses, date, null) - ├── lists all verses with exact KJV text - └── non-holiday template (thematic thread, cross-references) - -generateDevotional(prompt) [gpt-5.4] - └── weaves all verified verses into one devotional - -saveDevotional(message, date, testament, "multi", verses) -``` - -## Flow: Holiday (Always Multi-Verse) - -``` -getHoliday(today) - ├── checks Easter-based holidays (offset from computeEaster) - │ └── Shrove Tue, Ash Wed, Laetare Sun, Palm Sun, - │ Holy Mon, Spy Wed, Maundy Thu, Good Fri, - │ Holy Sat, Easter, Ascension, Pentecost, Trinity Sun - │ - ├── checks fixed-date holidays (month-day lookup) - │ └── New Year's, Epiphany Eve, Epiphany, Valentine's, - │ St. Patrick's, Spring Equinox, Earth Day, Flag Day, - │ Juneteenth, Summer Solstice, Independence Day, - │ Transfiguration, Patriot Day, Int'l Day of Peace, - │ Autumn Equinox, Michaelmas, Reformation Day, - │ All Saints', Veterans Day, Winter Solstice, - │ Christmas Eve, Christmas, New Year's Eve - │ - └── checks moveable holidays (computed per year) - └── MLK Day, Presidents' Day, Memorial Day, Mother's Day, - Father's Day, Labor Day, Baptism of Jesus, - World Day of Prayer, National Day of Prayer, - Election Day, Thanksgiving, Christ the King Sun, - 4 Advent Sundays - -determineDevotionalType(supabase, today, holiday) - └── holiday detected → always returns { type: "multi" } - -verseCount = random(2 or 3) - -selectMultiVerses(verseCount, holiday) [gpt-5-mini] - ├── prompt includes: holiday.themeHint - ├── "pick N verses related to [Holiday Name]" - └── verse selection is themed but different each year - -for each verse ref: - lookupVerseText(book, chapter, verse) [bible.json] - └── exact KJV text, tags stripped - -createMultiVersePrompt(resolvedVerses, date, holiday) - ├── lists all verses with exact KJV text - ├── includes holiday.themeHint - ├── title includes holiday name - └── modern relevance connects to holiday significance - -generateDevotional(prompt) [gpt-5.4] - -saveDevotional(message, date, testament, "multi", verses) -``` - -## Flow: Single Verse on a Holiday - -This happens when the rotation says "single" but it's a holiday. The holiday's curated verse pool is used instead of a random Bible verse. - -``` -getHoliday(today) → holiday detected - -determineDevotionalType() - └── holiday → always "multi" (overrides rotation) -``` - -So single-verse mode **never fires on holidays**. The rotation picks back up on the next non-holiday day based on yesterday's `devotional_type`. - ---- - -## Models - -| Model | Purpose | Max Tokens | -|-------|---------|------------| -| `gpt-5.4-mini` | Verse selection (multi-verse step 1) | 300 | -| `gpt-5.4` | Devotional writing (all modes) | 4000 | - -## Database Schema +Generates a daily KJV devotional, judges it, and stores the selected result. +iOS and Android read public fields through `get-daily-devotional`. + +## Request and rotation + +The caller supplies `SuperSecret`, matching the `SUPER_SECRET` environment +variable. Optional JSON fields: + +- `forDate`: UTC civil date (`YYYY-MM-DD`), default today. +- `force`: regenerate an existing date. The existing row is replaced only + after a publishable draft is available and saved successfully. + +Normal requests return existing content before making model calls. Generation +rotates independently through old-testament single, new-testament single, and +multi-verse; and through empathy, technical, narrative, practical, matt, josh, +lament, question and character tracks. Custom rows do not advance the track. +Holidays use one curated verse and retain holiday metadata through retries. + +Model IDs live in `index.ts`: `DEVOTIONAL_MODEL`, `VERSE_SELECTION_MODEL`, +and `JUDGE_MODEL`. Model, prompt, prompt version and token usage are stored +with each draft. Environment variables do not override model constants. + +## Selection and quality gate + +Random anchors are sampled uniformly from eligible paragraphs, so a book's +chance is proportional to its usable material. `_shared/verse-selection.ts` +filters fragments, very long paragraphs, and narrow patterns for census, +genealogy, itinerary and measurement passages (about 0.7% of the eligible +pool). The rules do not infer importance from proper names. + +Multi-verse selection chooses a random seed and asks for a companion from +another book. References must resolve to bundled KJV text. Both seed and +companion exclude previously attempted references. Invalid or repeated model +references are retried before spending another generation call. + +Each draft is judged on verse weight, faithfulness, substance, track fit and +craft. Code computes the weighted 0–10 score. A normal pass requires: + +- overall score at least 7.0; +- every criterion above 2 on its 1–5 scale; +- no hard-failure flags, such as misquoted scripture or factual errors. + +At most three drafts are generated on the same track/type/testament. All +selection, generation and judge calls share a 120-second deadline. Database +and Sentry requests have five-second limits, leaving time to persist results +before Supabase's request timeout. + +Every failed judgment is immediately written to `daily_devotional_rejections`, +before another model request. Audit errors are reported without discarding +completed drafts. The table has RLS enabled and no app-role table privileges. +A `run_id` and attempt number connect audit records to published metadata and +distinguish repeated invocations. + +Outcomes: + +- **Pass:** publish with judge scores and attempt count. +- **Exhaustion, retry error or deadline:** use the highest-scoring completed + draft with no hard failures and every criterion above 2. Mark + `judge.exhausted = true`, retain `judge.retry_error` if present, and report to + Sentry. This soft-score fallback remains audited because its judgment failed + the overall threshold. +- **No usable draft:** return 503, retain all failed judgments, and leave any + existing devotional untouched. Known errors are never a fallback. +- **Judge unavailable:** retain the established fail-open behavior: publish + the completed draft with `judge.result = null` and `judge.error` populated. + An unavailable judge is distinct from a known failed judgment. +- **First generation fails:** return 500 without changing existing content. ```sql -"Daily Devotional" ( - id bigint PRIMARY KEY, - created_at timestamptz DEFAULT now(), - message text DEFAULT '', -- markdown devotional - for_date date UNIQUE NOT NULL, -- one per day - testament text, -- 'old' or 'new' - devotional_type text DEFAULT 'single', -- 'single' or 'multi' - verses jsonb -- [{book, chapter, verse, testament}] -) -``` +select for_date, run_id, attempt, track, verses, judge_score, + judge->'hard_fails', judge->'reasons' +from daily_devotional_rejections order by created_at desc; -## Verse Accuracy - -All verse text comes from `bible.json` (bundled KJV data) via `lookupVerseText()`: -- Strips `...` red-letter tags -- ~89% of holiday verse references resolve directly (matching `startingVerse`) -- ~11% are mid-paragraph; fallback to hardcoded text in holiday definitions - -## Testing - -```bash -cd supabase/functions/daily-devotional -deno test holidays_test.ts --allow-read +select for_date, judge from "Daily Devotional" +where (judge->>'exhausted')::boolean or judge->'result' = 'null'::jsonb; ``` -18 tests covering: Easter computation, date helpers, holiday dates for 2026-2030, verse validity against bible.json, testament labels, collision detection, and model verification. +## Tests and evaluations -## Local Invocation - -```bash -make test_daily_devotional - -# or manually: -curl -X POST 'http://127.0.0.1:54321/functions/v1/daily-devotional' \ - -H 'Authorization: Bearer $SWIFTBIBLE_KEY' \ - -H 'SuperSecret: $SWIFTBIBLE_SUPERSECRET_KEY' +```sh +deno test --allow-all supabase/functions/ +deno check supabase/functions/daily-devotional/index.ts +OPENAI_API_KEY=... make eval-devotional-judge TRIALS=3 ``` + +The normal suite covers scoring, retry recovery, deadlines, real request +handler behavior with intercepted network requests, fresh verse selection, +holiday metadata, audit ordering, and bundled KJV fixture accuracy. It never +calls a live model or database. CI also checks request-handler types. + +Live evaluations are opt-in and bill model calls. They include the rejected +2026-10-11 Numbers 26:58 devotional, positive examples, invented scenes, +misreadings, theological errors and broken structure. The empathy and narrative writers +share their corrected examples with the positive evaluation fixtures in +`_shared/devotional-examples.ts`; the former empathy invented-scene example +remains as a negative control. Re-run evaluations when models, rubric or examples +change. Bump the corresponding prompt version when its prompt changes. + +## Deployment + +Merge to `master` to deploy through GitHub Actions. The Supabase deployment +workflow calls the migration workflow first and deploys functions only after +it succeeds. Production deployment runs are serialized. Do not deploy +functions or push migrations manually. diff --git a/supabase/functions/daily-devotional/evals/judge_cases.ts b/supabase/functions/daily-devotional/evals/judge_cases.ts new file mode 100644 index 0000000..2eef9d9 --- /dev/null +++ b/supabase/functions/daily-devotional/evals/judge_cases.ts @@ -0,0 +1,385 @@ +import { EMPATHY_EXAMPLE, NARRATIVE_EXAMPLE } from "../../_shared/devotional-examples.ts"; +// Labelled cases for the devotional judge (see ../judge_eval_test.ts). +// +// Each case is a devotional the judge must get right. `expect` is the +// publish decision a careful human editor would make; `hardFails` (optional) +// are flags the judge must raise for the decision to count as right for the +// right reason. +// +// When the judge gets a real day wrong, add that day here with the label it +// should have had. That is how this set is meant to grow: from production +// misses, not from invented edge cases. The first entry is the day that +// prompted the judge. +// +// Keep the verse text exact KJV (bible.json, tags stripped) so a +// "misquoted_scripture" flag is only ever about the devotional, never the +// fixture. + +import type { JudgeHardFail, JudgeVerse } from "../../_shared/devotional-judge.ts"; + +export interface JudgeCase { + id: string; + expect: "pass" | "fail"; + // Why the label is what it is. Printed when the judge disagrees. + why: string; + track: string; + verses: JudgeVerse[]; + holidayName?: string; + // At least one of these must be flagged. Omit when the fail should come + // from scores alone. + hardFails?: JudgeHardFail[]; + message: string; +} + +export const JUDGE_CASES: JudgeCase[] = [ + { + id: "prod-2026-10-11-numbers-26-58-lament", + expect: "fail", + why: + "Production miss that prompted the judge. A census line of Levite clan names given the lament track; the grief is imported from the next verse and the sections are one or two sentences each.", + track: "lament", + verses: [{ + book: "Numbers", + chapter: 26, + verse: 58, + text: + "These are the families of the Levites: the family of the Libnites, the family of the Hebronites, the family of the Mahlites, the family of the Mushites, the family of the Korathites. And Kohath begat Amram.", + }], + message: `# October 11, 2026 — Numbers 26:58: Names That Remain + +**A family can still be named after people in it have died.** + +> *"These are the families of the Levites: the family of the Libnites, the family of the Hebronites, the family of the Mahlites, the family of the Mushites, the family of the Korathites. And Kohath begat Amram."* +> **Numbers 26:58** + +## What is lost + +Aaron and Miriam are dead. The next verse names them among Amram's children. Their place in the family record remains; their living presence does not. + +## The passage does not tidy this + +The census continues through the families of Levi. Kohath begat Amram. The names are written down, and the people left behind must go on hearing them. This verse gives no account of what it was like to speak a dead person's name during the count. + +## Still speaking + +Father, I can say their names to You. I do not know what to do with the silence that follows. + +## A prayer + +*Father, hear me when I speak the names of those I miss. Stay near while I have no more words. Amen.*`, + }, + + { + id: "house-example-mark-1-35-empathy", + expect: "pass", + why: "The empathy track's own few-shot exemplar. If the judge fails this, the rubric disagrees with the house style.", + track: "empathy", + verses: [{ + book: "Mark", + chapter: 1, + verse: 35, + text: + "And in the morning, rising up a great while before day, he went out, and departed into a solitary place, and there prayed.", + }], + message: EMPATHY_EXAMPLE, + }, + + { + id: "invented-scene-mark-1-35", + expect: "fail", + why: "The former house example invents Jesus's movements and motives; polished prose must not excuse those claims.", + hardFails: ["factual_error"], + track: "empathy", + verses: [{ book: "Mark", chapter: 1, verse: 35, text: "And in the morning, rising up a great while before day, he went out, and departed into a solitary place, and there prayed." }], + message: `# May 15 — Mark 1:35: Before the world wakes + +**Why does the answer to the busiest day arrive at the quietest hour?** + +> *"And in the morning, rising up a great while before day, he went out, and departed into a solitary place, and there prayed."* +> **Mark 1:35** + +## The hour before the work + +The first sound is the door clicking shut behind him. Careful, as if afraid of waking the world. He stops on the threshold a moment, listening. Around him is the small hum of a town that has not woken up: a distant dog, the wind, nothing else. No crowd. No one needing to be healed yet. + +There is, for now, just him and the cold and the long walk to whatever solitary place he has in mind. + +## What he chose + +The Greek for "solitary place" is *eremos*, the same word the Gospels use for the wilderness where Jesus was tempted. Stripped down. Without props. + +He has just had what any of us would call a successful day. The whole city pressed at the door (Mark 1:33). Demons cast out, fevers gone. The kind of day a ministry would build a website around. + +And before any of it could harden into an identity, he leaves it. + +## Your version of the hour + +If you have ever felt like the day's noise starts the moment you open your eyes, you know how thin the margin gets. The hour Jesus chose is not magic. It is just the only hour the world has not yet asked for. + +You may not have a hillside. You may have ten minutes in a parked car before walking into the building. The order matters more than the place. + +## A prayer + +*Father, before the day asks me for anything, let me ask You first. Teach me the hour You chose. Make me unhurried in it. You are worth the full attention, even when I have none to spare. Amen.*`, + }, + + { + id: "house-example-luke-19-5-narrative", + expect: "pass", + why: "The narrative writer's actual example distinguishes Luke's account from inference and supplies no imagined physical details or motives.", + track: "narrative", + verses: [{ book: "Luke", chapter: 19, verse: 5, text: "And when Jesus came to the place, he looked up, and saw him, and said unto him, Zacchaeus, make haste, and come down; for to day I must abide at thy house." }], + message: NARRATIVE_EXAMPLE, + }, + + { + id: "good-lament-psalm-13-1", + expect: "pass", + why: + "A lament on a passage that is itself a lament. Does not resolve, by design. Guards against the judge marking the lament track down for lacking comfort.", + track: "lament", + verses: [{ + book: "Psalms", + chapter: 13, + verse: 1, + text: "How long wilt thou forget me, O LORD? for ever? how long wilt thou hide thy face from me?", + }], + message: `# March 3, 2026 — Psalms 13:1: How long + +**David asks the question four times and gets no answer in the asking.** + +> *"How long wilt thou forget me, O LORD? for ever? how long wilt thou hide thy face from me?"* +> **Psalms 13:1** + +## What is lost + +The thing David has lost is the sense of being seen. He does not say God is gone; he says God has forgotten him and turned His face away. The questions describe the pain of feeling forgotten. They do not establish that God has stopped attending to him. + +"How long" comes four times in two verses. Anyone who has waited on a diagnosis, a phone call, or a prodigal knows that a question repeated is not a question asked for information. It is the sound of a person who has run out of other things to say. + +## The passage does not tidy this + +David adds "for ever?" to his first question, and the psalm lets it stand. In this opening verse he does not yet turn to God's past faithfulness. The next verse names "sorrow in my heart daily." Scripture keeps this complaint in the hymnbook of Israel without a footnote explaining why the delay was good for him. + +The temptation, reading it, is to hurry on to verse 5 where he says he has trusted in God's mercy. That verse is real. So is this one, and it comes first. + +## Still speaking + +The address never changes. Every "how long" is aimed at "O LORD." The complaint is addressed to God. Even the question about being forgotten remains a form of speaking to Him. If that is where you are today, the psalm gives you permission to stay there and keep talking. + +## A prayer + +*Father, I do not know how long this will last, and I am tired of asking. I am asking You anyway. Do not forget me. Amen.*`, + }, + + { + id: "mtd-philippians-4-13-practical", + expect: "fail", + why: + "Reads Philippians 4:13 as a promise of personal achievement; the verse is Paul on contentment in want and plenty (4:11-12). Self-help language the guardrails ban.", + track: "practical", + verses: [{ + book: "Philippians", + chapter: 4, + verse: 13, + text: "I can do all things through Christ which strengtheneth me.", + }], + hardFails: ["theological_error", "factual_error"], + message: `# June 9, 2026 — Philippians 4:13: Nothing is out of reach + +**What would you attempt if you knew you could not fail?** + +> *"I can do all things through Christ which strengtheneth me."* +> **Philippians 4:13** + +## The goal in front of you + +The promotion. The marathon. The business plan sitting in a folder on your desktop. Most of us carry a dream we have quietly decided is too big. + +## What Paul promises + +Paul is clear: *all* things. The Greek *panta* leaves nothing out. Whatever you set your heart on, Christ supplies the strength to reach it. Paul wrote this as a man who had accomplished more than almost anyone in the ancient world, and he wants you to know the same power is yours. + +## Take the step + +So write the goal down today. Tell one person. Take the first step. You have got this, because God wants you to win and He has already given you everything you need to be your best self. + +## A prayer + +*Father, thank You that nothing is impossible for me. Give me the confidence to chase every dream You have put in my heart. Amen.*`, + }, + + { + id: "factual-error-1-kings-19-4-narrative", + expect: "fail", + why: + "Swaps Elijah for Elisha, sets the scene after Sinai (it is on the way there) and puts Ahab in pursuit; 1 Kings 19:2-3 has Jezebel's threat.", + track: "narrative", + verses: [{ + book: "1 Kings", + chapter: 19, + verse: 4, + text: + "But he himself went a day's journey into the wilderness, and came and sat down under a juniper tree: and he requested for himself that he might die; and said, It is enough; now, O LORD, take away my life; for I am not better than my fathers.", + }], + hardFails: ["factual_error"], + message: `# August 2, 2026 — 1 Kings 19:4: Under the juniper + +**The prophet who called down fire now asks to die.** + +> *"But he himself went a day's journey into the wilderness, and came and sat down under a juniper tree: and he requested for himself that he might die; and said, It is enough; now, O LORD, take away my life; for I am not better than my fathers."* +> **1 Kings 19:4** + +## The scene + +Elisha has just come down from Mount Sinai, where God spoke to him in the still small voice. King Ahab's soldiers are a day behind him. He walks until his legs give out and drops under the only shade in sight, a scrubby juniper. + +## What he says + +"It is enough." Elisha has seen fire fall on Carmel and rain end a drought, and none of it has changed Ahab. The Hebrew *rab* carries the sense of "too much," a weight he can no longer lift. + +## Where it meets you + +If you have ever done the hard, right thing and watched nothing change, you know this tree. God's answer, in the next verses, is not a lecture. It is bread, water, and sleep. + +## A prayer + +*Father, when I have nothing left, feed me before You send me. Amen.*`, + }, + + { + id: "generic-psalm-46-10-empathy", + expect: "fail", + why: + "Correct but empty. Says nothing about the verse that could not be said of any verse about rest; ignores the second half of the verse entirely; thin sections.", + track: "empathy", + verses: [{ + book: "Psalms", + chapter: 46, + verse: 10, + text: "Be still, and know that I am God: I will be exalted among the heathen, I will be exalted in the earth.", + }], + message: `# April 20, 2026 — Psalms 46:10: Be still + +**What would happen if you just stopped?** + +> *"Be still, and know that I am God: I will be exalted among the heathen, I will be exalted in the earth."* +> **Psalms 46:10** + +## Busy days + +Life is busy. There is always something to do and somewhere to be, and it can feel like there is never time to rest. + +## God's invitation + +God invites us to be still. When we slow down, we can know Him better. Stillness is a gift that God wants to give each of us, and it is always available. + +## Finding stillness + +Take a few moments today to be still before God. Turn off distractions and simply rest in His presence. You will be glad you did. + +## A prayer + +*Father, help me to be still and know that You are God. Amen.*`, + }, + + { + id: "itinerary-numbers-33-19-practical", + expect: "fail", + why: + "Two place names on a wilderness itinerary stretched into a sermon on 'seasons of moving' that the verse does not contain. Even competently written, the verse cannot carry it.", + track: "practical", + verses: [{ + book: "Numbers", + chapter: 33, + verse: 19, + text: "And they departed from Rithmah, and pitched at Rimmonparez.", + }], + hardFails: ["verse_cannot_carry_devotional", "theological_error", "factual_error"], + message: `# July 14, 2026 — Numbers 33:19: Pitching the tent again + +**What do you do on the day you have to start over somewhere new?** + +> *"And they departed from Rithmah, and pitched at Rimmonparez."* +> **Numbers 33:19** + +## The action + +Unpack one box today in the place you are in, even if you do not expect to stay. Israel did not know how long they would camp at Rimmonparez, and they pitched anyway. + +## Why it matters + +Rithmah may come from a word for the broom shrub, and Rimmonparez joins "pomegranate" to "breach." Every move God led them on carried a lesson: from the bitter shrub to the sweet fruit, from dryness to provision. God was teaching them that every departure leads somewhere better, and He is teaching you the same in your season of transition. + +## When it goes sideways + +Some moves feel like going backwards. Israel's did too. Trust that the next campsite is part of the plan, and keep pitching the tent. + +## A prayer + +*Father, wherever You move me next, help me settle in and trust You there. Amen.*`, + }, + + { + id: "spare-matt-luke-17-10", + expect: "pass", + why: + "The matt track is short and dry on purpose. Precise on the text, a couple of imperatives, no flourish. Guards against the judge rewarding length.", + track: "matt", + verses: [{ + book: "Luke", + chapter: 17, + verse: 10, + text: + "So likewise ye, when ye shall have done all those things which are commanded you, say, We are unprofitable servants: we have done that which was our duty to do.", + }], + message: `# September 8, 2026 — Luke 17:10: Duty done + +**Finishing the job does not put God in your debt.** + +> *"So likewise ye, when ye shall have done all those things which are commanded you, say, We are unprofitable servants: we have done that which was our duty to do."* +> **Luke 17:10** + +## The thing itself + +The dishes are done. Nobody noticed. You would like somebody to notice. + +## What it actually says + +Jesus has just described a servant who comes in from the field and is told to make supper before he eats (Luke 17:7-8). The master does not thank him for it. Then comes the turn: "So likewise ye." + +Look at the condition. "When ye shall have done all those things." Jesus is speaking about the servant who did everything, and even that servant has only done his duty. The word "unprofitable" does not mean worthless. It means he has produced nothing beyond what was already owed. + +That is hard to hear and good to know. Obedience is the floor. Grace is what God gives on top of it, and it is never wages. + +## So + +Do what you were told. Do it without keeping score. Stop waiting for the thank-you. Thank Him instead. + +## A prayer + +*Father, I have not given You anything You did not first give me. Keep me faithful in the ordinary work, and keep me from thinking it makes You owe me. Amen.*`, + }, + + { + id: "broken-format-lamentations-3-22", + expect: "fail", + why: + "Reasonable content, but no title line, no verse blockquote and no prayer; the app renders this skeleton and the reader gets none of it.", + track: "empathy", + verses: [{ + book: "Lamentations", + chapter: 3, + verse: 22, + text: "It is of the LORD's mercies that we are not consumed, because his compassions fail not.", + }], + hardFails: ["broken_format"], + message: `Lamentations 3:22 says that it is of the Lord's mercies that we are not consumed, because his compassions fail not. + +Jeremiah wrote this in the ruins of Jerusalem. The city had burned. The temple was gone. He had every reason to say God had abandoned His people, and for most of the book he says something close to it. + +Then, in the middle of the third chapter, he remembers something. The fact that anyone is still alive at all is mercy. The Hebrew *chesed*, often translated "mercies," is covenant loyalty, love that keeps its promises. + +If you are standing in your own ruins, this verse does not pretend the ruins are not there. It just notices that you are still standing in them.`, + }, +]; diff --git a/supabase/functions/daily-devotional/handler_test.ts b/supabase/functions/daily-devotional/handler_test.ts new file mode 100644 index 0000000..e670836 --- /dev/null +++ b/supabase/functions/daily-devotional/handler_test.ts @@ -0,0 +1,315 @@ +// Exercise the real handler, Supabase client and model request construction. +// The fetch boundary is intercepted: no production data or model traffic. +import { handleRequest } from "./index.ts"; +import { JUDGE_CRITERIA } from "../_shared/devotional-judge.ts"; + +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(message); +} +// deno-lint-ignore no-explicit-any +type Row = Record; +interface Scenario { + judgments: Array; + multi?: boolean; + holiday?: boolean; + failGeneration?: number; + failSelectionAfter?: number; + repeatedCompanion?: boolean; + auditError?: boolean; + cached?: boolean; + denied?: boolean; +} + +Deno.test("real devotional handler recovery and persistence", async (t) => { + const originalFetch = globalThis.fetch; + const originalRandom = Math.random; + const originalLog = console.log; + const env = { + SUPABASE_URL: "http://review-database.invalid", + SUPABASE_SERVICE_ROLE_KEY: "review-key", + SUPER_SECRET: "review-secret", + OPENAI_API_KEY: "review-key", + }; + const previous = Object.fromEntries( + Object.keys(env).map((key) => [key, Deno.env.get(key)]), + ); + for (const [key, value] of Object.entries(env)) Deno.env.set(key, value); + console.log = () => {}; + + async function run(scenario: Scenario) { + let generated = 0, selected = 0, judged = 0; + const writes: Array<{ table: string; rows: Row | Row[] }> = []; + const order: string[] = []; + const signals: AbortSignal[] = []; + // A constant RNG recreates the original duplicate seed bug reliably. + Math.random = () => 0.2; + const json = (value: unknown, status = 200) => + new Response(JSON.stringify(value), { + status, + headers: { "Content-Type": "application/json" }, + }); + globalThis.fetch = (input, init) => { + const url = new URL( + input instanceof Request ? input.url : input.toString(), + ); + if (url.hostname === "review-database.invalid") { + if ((init?.method ?? "GET") === "GET") { + const select = url.searchParams.get("select"); + if (select === "message") { + return Promise.resolve( + json(scenario.cached ? { message: "existing devotional" } : null), + ); + } + if (select === "track") { + return Promise.resolve(json({ track: "josh" })); + } + if (select === "testament,devotional_type") { + return Promise.resolve( + json({ + testament: "new", + devotional_type: scenario.multi ? "single" : "multi", + }), + ); + } + throw new Error(`Unexpected database read: ${select}`); + } + assert(init?.signal, "database writes must have a timeout"); + const table = decodeURIComponent(url.pathname.split("/").at(-1)!); + order.push(table); + writes.push({ table, rows: JSON.parse(init!.body as string) }); + return Promise.resolve( + scenario.auditError && table === "daily_devotional_rejections" + ? json({ message: "audit unavailable" }, 500) + : new Response(null, { status: 201 }), + ); + } + assert( + url.hostname === "api.openai.com", + `Unexpected network request: ${url.hostname}`, + ); + assert(init?.signal, "every model request must have a time limit"); + signals.push(init.signal); + const body = JSON.parse(init.body as string); + const system = body.messages[0].content; + let content: string; + if (system.includes("strict editor")) { + const value = scenario.judgments[judged++]; + order.push("judge"); + if (value === "down") { + return Promise.resolve(json({ error: "judge down" }, 503)); + } + assert(value !== undefined, "unexpected judge call"); + content = JSON.stringify({ + scores: Object.fromEntries( + JUDGE_CRITERIA.map(( + c, + ) => [c, value === "hard" ? c === "faithfulness" ? 2 : 5 : value]), + ), + hard_fails: value === "hard" ? ["misquoted_scripture"] : [], + reasons: [], + }); + } else if (system.includes("verse selector")) { + selected++; + order.push("selection"); + if ( + scenario.failSelectionAfter !== undefined && + selected > scenario.failSelectionAfter + ) return Promise.resolve(json({ error: "selection down" }, 503)); + const companion = scenario.repeatedCompanion || selected === 1 + ? { book: "Matthew", chapter: 6, verse: 33 } + : { book: "John", chapter: 3, verse: 16 }; + content = JSON.stringify({ verses: [companion] }); + } else { + generated++; + order.push(`generate-${generated}`); + if (generated === scenario.failGeneration) { + return Promise.resolve(json({ error: "generation down" }, 503)); + } + content = `# Draft ${generated}`; + } + return Promise.resolve( + json({ + choices: [{ message: { content } }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }), + ); + }; + const response = await handleRequest( + new Request("http://review-handler.invalid", { + method: "POST", + headers: { + "Content-Type": "application/json", + SuperSecret: scenario.denied ? "wrong" : "review-secret", + }, + body: JSON.stringify({ + forDate: scenario.holiday ? "2026-12-25" : "2026-10-12", + }), + }), + ); + const published = writes.find((w) => w.table === "Daily Devotional") + ?.rows as Row | undefined; + const rejected = writes.filter((w) => + w.table === "daily_devotional_rejections" + ).flatMap((w) => w.rows); + return { + status: response.status, + body: await response.text(), + published, + rejected, + generated, + selected, + order, + signals, + }; + } + try { + await t.step( + "authorization and cached content make no model calls", + async () => { + const denied = await run({ judgments: [], denied: true }); + assert( + denied.status === 403 && denied.generated === 0, + "unauthorized request generated content", + ); + const cached = await run({ judgments: [], cached: true }); + assert( + cached.body === "existing devotional" && cached.generated === 0, + "cache should be returned unchanged", + ); + }, + ); + await t.step( + "single retry uses a new passage and persists failure before generating again", + async () => { + const r = await run({ judgments: [2, 4] }); + assert( + r.status === 200 && r.published?.judge.result.passed && + r.rejected.length === 1, + "retry did not publish the passing draft", + ); + assert( + JSON.stringify(r.published.verses) !== + JSON.stringify(r.rejected[0].verses), + "retry reused the rejected verse", + ); + assert( + r.order.indexOf("daily_devotional_rejections") < + r.order.indexOf("generate-2"), + "audit happened too late", + ); + assert( + r.rejected[0].run_id === r.published.judge.run_id, + "audit and published result must identify the same run", + ); + }, + ); + for (const stage of ["generation", "selection"]) { + await t.step( + `a later ${stage} outage preserves the earlier usable draft and audit`, + async () => { + const r = await run({ + judgments: [3], + ...(stage === "generation" + ? { failGeneration: 2 } + : { multi: true, failSelectionAfter: 1 }), + }); + assert( + r.status === 200 && r.published?.judge.exhausted && + r.rejected.length === 1, + "fallback or audit lost", + ); + assert( + r.published.judge.retry_error.includes(`${stage}`), + "retry error missing from row", + ); + }, + ); + } + await t.step( + "known errors cannot win fallback ranking or overwrite existing content", + async () => { + const r = await run({ judgments: [3, "hard", 3] }); + assert( + r.status === 200 && r.published?.judge.attempt === 1 && + r.rejected.length === 3, + "hard failure selected or audit missing", + ); + const blocked = await run({ judgments: ["hard", "hard", "hard"] }); + assert( + blocked.status === 503 && !blocked.published && + blocked.rejected.length === 3, + "unsafe draft published", + ); + }, + ); + await t.step( + "multi retries exclude both prior seed and companion", + async () => { + const r = await run({ judgments: [2, 4], multi: true }); + assert( + r.status === 200 && r.published && r.rejected.length === 1, + "multi retry failed", + ); + const previous = new Set( + r.rejected[0].verses.map((v: Row) => + `${v.book} ${v.chapter}:${v.verse}` + ), + ); + assert( + r.published.verses.every((v: Row) => + !previous.has(`${v.book} ${v.chapter}:${v.verse}`) + ), + "multi retry reused a reference", + ); + const repeat = await run({ + judgments: [3], + multi: true, + repeatedCompanion: true, + }); + assert( + repeat.generated === 1 && repeat.published?.judge.retry_error, + "repeated companion reached generation", + ); + }, + ); + await t.step( + "holiday retry advances to an unused curated verse", + async () => { + const r = await run({ judgments: [2, 4], holiday: true }); + assert( + r.status === 200 && r.published && + r.published.holiday_name?.startsWith("Christmas"), + "holiday metadata lost", + ); + assert( + JSON.stringify(r.published.verses) !== + JSON.stringify(r.rejected[0].verses), + "holiday verse reused", + ); + }, + ); + await t.step( + "judge and audit outages preserve documented availability behavior", + async () => { + const r = await run({ judgments: ["down"] }); + assert( + r.status === 200 && r.published?.judge.result === null && + r.published.judge.error, + "judge outage metadata missing", + ); + const audit = await run({ judgments: [2, 4], auditError: true }); + assert( + audit.status === 200 && audit.published?.judge.result.passed, + "audit failure blocked passing draft", + ); + }, + ); + } finally { + globalThis.fetch = originalFetch; + Math.random = originalRandom; + console.log = originalLog; + for (const [key, value] of Object.entries(previous)) { + value === undefined ? Deno.env.delete(key) : Deno.env.set(key, value); + } + } +}); diff --git a/supabase/functions/daily-devotional/index.ts b/supabase/functions/daily-devotional/index.ts index 9c55464..accf75f 100644 --- a/supabase/functions/daily-devotional/index.ts +++ b/supabase/functions/daily-devotional/index.ts @@ -1,13 +1,24 @@ // Setup type definitions for built-in Supabase Runtime APIs import "jsr:@supabase/functions-js/edge-runtime.d.ts"; -import { createClient } from "jsr:@supabase/supabase-js@2"; +import { EMPATHY_EXAMPLE, NARRATIVE_EXAMPLE } from "../_shared/devotional-examples.ts"; +import { createClient, type SupabaseClient } from "jsr:@supabase/supabase-js@2"; import { initSentry, captureException } from "../_shared/sentry.ts"; -import { selectRandomVerse as sharedSelectRandomVerse } from "../_shared/verse-selection.ts"; +import { selectRandomVerse as sharedSelectRandomVerse, verseKey, unsuitableReason } from "../_shared/verse-selection.ts"; import { PROMPT_THEOLOGY_GUARDRAILS, PROMPT_MATT_RULES, PROMPT_JOSH_RULES, } from "../_shared/devotional-voice.ts"; +import { + buildJudgePrompt, + callJudgeModel, + generateWithJudge, + JUDGE_MAX_ATTEMPTS, + GENERATION_BUDGET_MS, + parseJudgeResponse, + type JudgedAttempt, + type JudgeResult, +} from "../_shared/devotional-judge.ts"; // ─── Types ────────────────────────────────────────────────────────── @@ -179,6 +190,11 @@ const OPENAI_CHAT_URL = "https://api.openai.com/v1/chat/completions"; // column so the iOS and Android apps can attribute it accurately. const DEVOTIONAL_MODEL = "gpt-6-sol"; const VERSE_SELECTION_MODEL = "gpt-5.6-luna"; +// Grades each draft before it is saved (see _shared/devotional-judge.ts). +// Same tier as the writer: a weaker grader misses the subtle failures +// (stretched verses, quiet misreadings) the gate exists to catch, and at one +// short call per attempt the cost is a fraction of generation. +const JUDGE_MODEL = "gpt-6-sol"; // Two parallel prompt tracks rotate randomly to vary the daily devotional // voice. The "Daily Devotional" table's prompt_version column records @@ -211,9 +227,9 @@ const VERSE_SELECTION_MODEL = "gpt-5.6-luna"; // than receiving it. The narrative few-shot (Zacchaeus) was // re-aimed Father-ward to match, since few-shots steer harder than // rules. -const EMPATHY_PROMPT_VERSION = "empathy-v8"; +const EMPATHY_PROMPT_VERSION = "empathy-v9"; const TECHNICAL_PROMPT_VERSION = "technical-v3"; -const NARRATIVE_PROMPT_VERSION = "narrative-v3"; +const NARRATIVE_PROMPT_VERSION = "narrative-v4"; const PRACTICAL_PROMPT_VERSION = "practical-v4"; const MATT_PROMPT_VERSION = "matt-v1"; const JOSH_PROMPT_VERSION = "josh-v1"; @@ -267,6 +283,7 @@ interface TokenUsage { interface DevotionalUsage { generation: TokenUsage; selection?: TokenUsage; + judge?: TokenUsage; } function extractUsage( @@ -1058,24 +1075,15 @@ function getHolidayInternal(date: Date): Holiday | null { // Verse selection lives in ../_shared/verse-selection.ts so it can be // imported and tested directly; this file cannot (top-level Deno.serve). -function selectRandomVerse(testament: "old" | "new"): SelectedVerse { +function selectRandomVerse(testament: "old" | "new", excluded: ReadonlySet = new Set()): SelectedVerse { return sharedSelectRandomVerse( testament === "old" ? OT_BOOKS : NT_BOOKS, testament, testament === "old" ? "Psalms" : "John", + excluded, ); } -function selectVerse( - targetTestament: "old" | "new", - holiday: Holiday | null -): SelectedVerse { - if (holiday && holiday.verses.length > 0) { - return holiday.verses[Math.floor(Math.random() * holiday.verses.length)]; - } - return selectRandomVerse(targetTestament); -} - // ─── Date formatting ──────────────────────────────────────────────── function getFormattedDate( @@ -1169,38 +1177,7 @@ const PROMPT_AI_TELLS = `AVOID (AI tells; strict) - "In a world where..." / "In our busy lives..." / "Let us not forget..." / "We must remember..." - Generic "may you..." benedictions, moralistic call-outs, and seminary tone.`; -const PROMPT_FEW_SHOT = `# May 15 — Mark 1:35: Before the world wakes - -**Why does the answer to the busiest day arrive at the quietest hour?** - -> *"And in the morning, rising up a great while before day, he went out, and departed into a solitary place, and there prayed."* -> **Mark 1:35** - -## The hour before the work - -The first sound is the door clicking shut behind him. Not a slam. Careful, as if afraid of waking the world. He stops on the threshold a moment, listening. Around him is the small hum of a town that has not woken up: a distant dog, the wind, nothing else. No crowd. No one needing to be healed. Not yet. - -There is, for now, just him, and the cold, and the dark, and the long walk to whatever solitary place he has in mind. - -## What he chose - -The Greek for "solitary place" is *eremos*, the same word the Gospels use for the wilderness where Jesus was tempted. Not just "alone." Stripped down. Without props. - -He has just had what any of us would call a successful day. The whole city pressed at the door (Mark 1:33). Demons cast out. Fevers gone. The kind of day a ministry would build a website around. - -And before any of it could harden into an identity, he leaves it. - -## Your version of the hour - -If you have ever felt like the day's noise starts the moment you open your eyes, you know how thin the margin gets. The hour Jesus chose is not magic. It is just the only hour the world has not yet asked for. - -You may not have a hillside. You may have ten minutes in a parked car before walking into the building. - -The geography is not the point. The order is. - -## A prayer - -*Father, before the day asks me for anything, let me ask You first. Teach me the hour You chose. Make me unhurried in it. You are worth the full attention, even when I have none to spare. Amen.*`; +const PROMPT_FEW_SHOT = EMPATHY_EXAMPLE; function createPrompt( verse: SelectedVerse, @@ -1255,7 +1232,10 @@ MARKDOWN OUTPUT (use exactly this skeleton; copy the verse text verbatim) async function selectMultiVerses( count: number, - holiday: Holiday | null + holiday: Holiday | null, + excluded: ReadonlySet = new Set(), + signal?: AbortSignal, + firstTestament: "old" | "new" = Math.random() < 0.5 ? "old" : "new", ): Promise<{ verses: Array<{ book: string; chapter: number; verse: number }>; usage: TokenUsage; @@ -1265,8 +1245,7 @@ async function selectMultiVerses( if (!apiKey) throw new Error("Missing OPENAI_API_KEY env var"); // Step 1: Pick the first verse randomly (guarantees variety) - const firstTestament: "old" | "new" = Math.random() < 0.5 ? "old" : "new"; - const seedVerse = selectRandomVerse(firstTestament); + const seedVerse = selectRandomVerse(firstTestament, excluded); console.log( `Multi-verse seed (random): ${seedVerse.book} ${seedVerse.chapter}:${seedVerse.verse}` ); @@ -1287,6 +1266,7 @@ Rules: - The companion should add a new dimension to the theme — not just echo the same idea - Use exact book names as they appear in the KJV (e.g. "Psalms" not "Psalm", "1 Corinthians" not "First Corinthians", "Song of Solomon" not "Song of Songs") - Only use books from the 66-book Protestant canon +- Do not reuse any of these previously tried references: ${[...excluded].join(", ") || "none"} Return ONLY a JSON object in this exact format: { @@ -1296,11 +1276,10 @@ Return ONLY a JSON object in this exact format: }`; const maxRetries = 2; - let content: string | undefined; let lastError: Error | undefined; - let usage: TokenUsage = extractUsage(null, VERSE_SELECTION_MODEL); for (let attempt = 0; attempt <= maxRetries; attempt++) { + signal?.throwIfAborted(); if (attempt > 0) { const delay = attempt * 1000; console.log(`selectMultiVerses retry ${attempt}/${maxRetries} after ${delay}ms`); @@ -1327,53 +1306,49 @@ Return ONLY a JSON object in this exact format: response_format: { type: "json_object" }, max_completion_tokens: 300, }), - signal: AbortSignal.timeout(30000), + signal: AbortSignal.any([AbortSignal.timeout(30000), ...(signal ? [signal] : [])]), }); if (!response.ok) { - let errorText: string | undefined; - try { - errorText = JSON.stringify(await response.json()); - } catch (_) { - errorText = await response.text(); - } + const errorText = await response.text(); lastError = new Error(`Verse selection failed (${response.status}): ${errorText}`); continue; } const data = await response.json(); console.log("selectMultiVerses response:", JSON.stringify(data, null, 2)); - content = data?.choices?.[0]?.message?.content; - usage = extractUsage(data, VERSE_SELECTION_MODEL); + const content = data?.choices?.[0]?.message?.content; + const usage = extractUsage(data, VERSE_SELECTION_MODEL); if (!content) { lastError = new Error(`Empty verse selection response: ${JSON.stringify(data)}`); continue; } - break; + const parsed = JSON.parse(content); + if (!Array.isArray(parsed.verses) || parsed.verses.length !== companionCount) { + throw new Error("Invalid verse selection count"); + } + const seen = new Set([...excluded, verseKey(seedVerse)]); + for (const ref of parsed.verses) { + if (!ref || typeof ref.book !== "string" || !Number.isInteger(ref.chapter) || !Number.isInteger(ref.verse)) { + throw new Error("Invalid verse reference"); + } + const text = lookupVerseText(ref.book, ref.chapter, ref.verse); + if (seen.has(verseKey(ref)) || ref.book === seedVerse.book || !text || unsuitableReason(text)) { + throw new Error(`Unusable or repeated companion: ${verseKey(ref)}`); + } + seen.add(verseKey(ref)); + } + return { + verses: [{ book: seedVerse.book, chapter: seedVerse.chapter, verse: seedVerse.verse }, ...parsed.verses], + usage, prompt, + }; } catch (err) { lastError = err instanceof Error ? err : new Error(String(err)); } } - if (!content) throw lastError ?? new Error("Verse selection failed after retries"); - - const parsed = JSON.parse(content); - if (!Array.isArray(parsed.verses) || parsed.verses.length === 0) { - throw new Error("Invalid verse selection format"); - } - - // Combine: random seed verse + GPT companion(s) - const allVerses = [ - { book: seedVerse.book, chapter: seedVerse.chapter, verse: seedVerse.verse }, - ...parsed.verses, - ]; - - console.log( - `Verse selection: ${allVerses.map((v: { book: string; chapter: number; verse: number }) => `${v.book} ${v.chapter}:${v.verse}`).join(", ")}` - ); - - return { verses: allVerses, usage, prompt }; + throw lastError ?? new Error("Verse selection failed after retries"); } // ─── Multi-verse prompt creation ──────────────────────────────────── @@ -1536,16 +1511,16 @@ ${PROMPT_AI_TELLS} // scene with Jesus" framings — kept observational, not invocational. const PROMPT_NARRATIVE_RULES = `VOICE — narrative track -- Drop the reader inside the biblical scene. Sensory, observational, alive. -- Present tense, third-person observer ("Peter sees the door swing shut...") or limited second-person ("You're standing at the edge of the crowd..."). -- Anchor concretely in 1st-century reality: heat, dust, lamp oil, crowd noise, the smell of fish, the sound of sandals on stone. Geographical and cultural inference is allowed. +- Retell the biblical scene using the concrete actions and details scripture actually supplies. Be vivid through those details, not through invented scenery. +- Present tense, third-person observer. Distinguish the anchor verse from any surrounding passage you use. +- Stay in the period and place of the selected passage, whether Old or New Testament. Do not add dust, sounds, clothing, gestures or physical sensations that the passage does not report. If background is needed, label it as background rather than an observed event. - DO NOT invent dialogue, internal monologue, or character motivations that scripture doesn't supply. If a character doesn't say it in the text, don't put words in their mouth. If we don't know what someone thought or felt, don't tell us they thought or felt it. - One scene, not a montage. Hold the camera in one place. - No anachronisms. No phones, no traffic, no "in our 21st-century lives." - Avoid Ignatian "place yourself in the scene with Jesus and ask Him a question" framings — keep it observational, not invocational. STRUCTURE (four beats) -1. Scene — open inside the moment. Describe what a careful observer would have seen and heard. Build the room with specific, grounded detail (the rope coiled by the boat, the sweat on a forehead, the shadow of a tree). Vary sentence rhythm. +1. Scene — open with the people, setting and actions the text actually names. If the anchor is a saying or poem, follow its own imagery and speaker; do not invent a historical scene around it. Vary sentence rhythm. 2. The moment the verse lands — quote the verse in a blockquote with bolded reference. Place it in the action. Why these specific words, in this specific moment. One short Hebrew/Greek nuance is allowed if a word genuinely opens the scene; otherwise skip. 3. And now — short bridge that pulls the scene into the reader's life. Honor that the cost back then echoes the cost now. Don't preach; observe. End on the real difficulty, not a tidy bow. 4. Prayer — short open-handed italic paragraph using *single asterisks* (NOT a blockquote — the iOS verse-link detector treats blockquotes as verse references). Address the Father by default; address Jesus directly ("Lord Jesus") only when the verse itself models prayer or a cry to Jesus, and do not address the Holy Spirit. End with "Amen." @@ -1564,40 +1539,7 @@ AVOID // Few-shot example for the narrative track. Luke 19:5 (Zacchaeus) — picked // because the scripture supplies enough scene material (sycamore, crowd, // "looked up") that the example can be vivid without inventing anything. -const PROMPT_NARRATIVE_FEW_SHOT = `# October 12 — Luke 19:5: Under the sycamore - -**The tree is the only place left.** - -> *"And when Jesus came to the place, he looked up, and saw him, and said unto him, Zacchaeus, make haste, and come down; for to day I must abide at thy house."* -> **Luke 19:5** - -## The tree - -The road into Jericho is dust and sandals. The crowd ahead is thick. Shoulders, robes, voices climbing over each other. Somewhere in the middle of it, the rabbi is moving slowly toward the city gate. - -Behind the crowd, a short man has run ahead. His tunic is hiked up. His sandals slap the stones. He is not used to running like this. - -A sycamore stands beside the road, ancient and wide, the branches starting low. He climbs. The bark scrapes his palms. - -From up here he can see the road. He can see the back of the rabbi's head moving through the press. He cannot, from up here, be seen. That's the point. A tax collector is not loved in Jericho. A short, rich tax collector is the kind of man who watches from above. - -## The moment - -The crowd reaches the tree. The rabbi stops walking. He looks up. - -The Greek behind "must" is *dei*, which carries the weight of necessity. Today. Your house. - -The words are said publicly. Anyone in the crowd could have heard them. The branches above Jericho stop being a hiding place. They become a name spoken out loud in front of everyone who hates him. - -## And now - -You may know the version of climbing into a tree. The corner of a room where the conversation can't reach. The polite distance from the people who would not be glad to see you. The carefully arranged life where you are not, exactly, available. - -Christ has a habit of stopping under exactly that tree. - -## A prayer - -*Father, You sent Your Son to find Zacchaeus in his hiding place and call him down by name. Find the tree I am in. Call me out of it, and give me the strength to come down. Amen.*`; +const PROMPT_NARRATIVE_FEW_SHOT = NARRATIVE_EXAMPLE; function createNarrativePrompt( verse: SelectedVerse, @@ -1891,7 +1833,8 @@ ${PROMPT_AI_TELLS} // ─── Devotional generation (OpenAI) ───────────────────────────────── async function generateDevotional( - prompt: string + prompt: string, + signal?: AbortSignal, ): Promise<{ message: string; usage: TokenUsage }> { const apiKey = Deno.env.get("OPENAI_API_KEY"); if (!apiKey) throw new Error("Missing OPENAI_API_KEY env var"); @@ -1915,7 +1858,7 @@ async function generateDevotional( ], max_completion_tokens: 4000, }), - signal: AbortSignal.timeout(120000), + signal: AbortSignal.any([AbortSignal.timeout(120000), ...(signal ? [signal] : [])]), }); if (!response.ok) { @@ -1952,6 +1895,7 @@ function createSupabaseClient() { return createClient( Deno.env.get("SUPABASE_URL") ?? "", Deno.env.get("SUPABASE_SERVICE_ROLE_KEY") ?? "", + { global: { fetch: (input, init) => fetch(input, { ...init, signal: AbortSignal.any([AbortSignal.timeout(5000), ...(init?.signal ? [init.signal] : [])]) }) } }, ); } @@ -2155,7 +2099,7 @@ function nextTrack(prev: string | null | undefined): Track { } async function determineDevotionalType( - supabase: ReturnType, + supabase: SupabaseClient, today: Date, holiday: Holiday | null ): Promise<{ @@ -2238,7 +2182,7 @@ interface PromptCapture { } async function saveDevotional( - supabase: ReturnType, + supabase: SupabaseClient, message: string, forDate: string, testament: string, @@ -2247,7 +2191,8 @@ async function saveDevotional( model: string, usage: DevotionalUsage, promptCapture: PromptCapture, - themeMetadata: ThemeMetadata = {} + themeMetadata: ThemeMetadata = {}, + judge: StoredJudge | null = null ): Promise { const versesJson = verses.map((v) => ({ book: v.book, @@ -2273,14 +2218,184 @@ async function saveDevotional( holiday_name: themeMetadata.holidayName ?? null, holiday_url: themeMetadata.holidayUrl ?? null, anchor_verse: themeMetadata.anchorVerse ?? null, + judge, }, { onConflict: "for_date" } ); if (error) throw error; } +// ─── Quality gate (judge + regenerate) ────────────────────────────── + +// One verse choice plus the prompt built from it. A retry builds a new one on +// the same track, type and testament, so "the same theme from a different +// verse" is literal: only the scripture changes. +interface Candidate { + verses: SelectedVerse[]; + prompt: string; + verseSelectionPrompt: string | null; + selectionUsage?: TokenUsage; +} + +interface Generation { + message: string; + usage: TokenUsage; + judgeUsage?: TokenUsage; +} + +// Judge result as persisted on the published row (Daily Devotional.judge). +// passed=false here only ever means every attempt failed and this was the +// best of them; null judge means the grader was unavailable. +interface StoredJudge { + run_id: string; + retry_error?: string; + model: string; + attempt: number; + attempts: number; + exhausted: boolean; + result: JudgeResult | null; + error?: string; +} + +async function buildCandidate( + devotionalType: "single" | "multi", + targetTestament: "old" | "new", + track: Track, + holiday: Holiday | null, + formattedDate: string, + tried: Candidate[], + signal?: AbortSignal, +): Promise { + const used = new Set(tried.flatMap((c) => c.verses.map(verseKey))); + + if (devotionalType === "multi") { + // Both seed and companion exclude every previously attempted reference. + const { verses: verseRefs, usage, prompt: selPrompt } = + await selectMultiVerses(2, null, used, signal, targetTestament); + const verses = verseRefs.map((ref) => { + const text = lookupVerseText(ref.book, ref.chapter, ref.verse); + const isOT = OT_BOOKS.some((b) => b.name === ref.book); + return { + book: ref.book, + chapter: ref.chapter, + verse: ref.verse, + text: text || `[${ref.book} ${ref.chapter}:${ref.verse}]`, + testament: isOT ? ("old" as const) : ("new" as const), + }; + }); + return { + verses, + prompt: buildMultiPrompt(track, verses, formattedDate, holiday), + verseSelectionPrompt: selPrompt, + selectionUsage: usage, + }; + } + + // Single: holidays draw from their curated pool first; once that is spent + // on failed attempts, fall back to the ordinary testament pool. + let verse: SelectedVerse | undefined; + if (holiday) { + const fresh = holiday.verses.filter((v) => !used.has(verseKey(v))); + if (fresh.length > 0) { + verse = { ...fresh[Math.floor(Math.random() * fresh.length)] }; + const resolvedText = lookupVerseText(verse.book, verse.chapter, verse.verse); + if (resolvedText) verse.text = resolvedText; + } + } + verse ??= selectRandomVerse(targetTestament, used); + + return { + verses: [verse], + prompt: buildSinglePrompt(track, verse, formattedDate, holiday), + verseSelectionPrompt: null, + }; +} + +async function judgeDraft( + candidate: Candidate, + generation: Generation, + track: Track, + holiday: Holiday | null, + signal?: AbortSignal, +): Promise { + const apiKey = Deno.env.get("OPENAI_API_KEY"); + if (!apiKey) throw new Error("Missing OPENAI_API_KEY env var"); + const { content, data } = await callJudgeModel({ + apiKey, + model: JUDGE_MODEL, + url: OPENAI_CHAT_URL, + signal, + prompt: buildJudgePrompt({ + message: generation.message, + track, + verses: candidate.verses, + holidayName: holiday?.name ?? null, + }), + }); + // deno-lint-ignore no-explicit-any + generation.judgeUsage = extractUsage(data as any, JUDGE_MODEL); + const result = parseJudgeResponse(content); + console.log("Judge result:", JSON.stringify(result)); + return result; +} + +// Persist every failed judgment before retrying. A soft-score fallback can +// later be published; run_id and attempt connect its audit to the final row. +// A write failure is reported without discarding the completed draft. +async function recordRejections( + supabase: SupabaseClient, + forDate: string, + devotionalType: string, + testament: string, + track: Track, + promptVersion: string, + holiday: Holiday | null, + rejected: JudgedAttempt[], + runId: string, +): Promise { + if (rejected.length === 0) return; + const rows = rejected.map((a) => ({ + run_id: runId, + for_date: forDate, + attempt: a.attempt, + track, + prompt_version: promptVersion, + devotional_type: devotionalType, + testament, + verses: a.candidate.verses.map((v) => ({ + book: v.book, + chapter: v.chapter, + verse: v.verse, + testament: v.testament, + })), + holiday_name: holiday?.name ?? null, + message: a.generation.message, + prompt: a.candidate.prompt, + verse_selection_prompt: a.candidate.verseSelectionPrompt, + model: DEVOTIONAL_MODEL, + usage: { + generation: a.generation.usage, + ...(a.candidate.selectionUsage ? { selection: a.candidate.selectionUsage } : {}), + ...(a.generation.judgeUsage ? { judge: a.generation.judgeUsage } : {}), + }, + judge_model: JUDGE_MODEL, + judge_version: a.judge?.version ?? null, + judge_score: a.judge?.overall ?? null, + judge_threshold: a.judge?.threshold ?? null, + judge: a.judge, + })); + const { error } = await supabase.from("daily_devotional_rejections").insert(rows); + if (error) { + console.error("[daily-devotional] failed to record rejected drafts", error); + await captureException(error, { + functionName: "daily-devotional", + tags: { stage: "record-rejections" }, + }); + } +} + async function fetchExistingDevotional( - supabase: ReturnType, + supabase: SupabaseClient, forDate: string ): Promise { const { data, error } = await supabase @@ -2301,9 +2416,9 @@ async function fetchExistingDevotional( // ─── Main handler ─────────────────────────────────────────────────── -initSentry("daily-devotional"); - -Deno.serve(async (req) => { +export async function handleRequest(req: Request): Promise { + const signal = AbortSignal.timeout(GENERATION_BUDGET_MS); + const runId = crypto.randomUUID(); // The shared secret is stored on the function as `SUPER_SECRET`. The caller // (daily cron) sends it in the `SuperSecret` header. Reading any other env // name leaves configuredSecret empty and 403s every request. @@ -2381,59 +2496,63 @@ Deno.serve(async (req) => { `Devotional type: ${devotionalType}, testament: ${targetTestament}, track: ${track} (${promptVersion})` ); - // Select verse(s) and create prompt - let prompt: string; - let verseSelectionPrompt: string | null = null; - let versesUsed: SelectedVerse[]; - let selectionUsage: TokenUsage | undefined; - - if (devotionalType === "multi") { - // Non-holiday multi-verse: random seed + GPT companion - const { verses: verseRefs, usage, prompt: selPrompt } = - await selectMultiVerses(2, null); - selectionUsage = usage; - verseSelectionPrompt = selPrompt; + // Select verse(s), generate, and judge. A draft below the judge's + // threshold is set aside and the same track/type/testament is tried + // again from a different verse (see _shared/devotional-judge.ts). + const tryCandidate = (tried: Candidate[]) => + buildCandidate(devotionalType, targetTestament, track, holiday, formatted, tried, signal); + + const outcome = await generateWithJudge({ + first: await tryCandidate([]), + next: tryCandidate, + generate: async (candidate) => { + console.log( + `Verses: ${candidate.verses.map(verseKey).join(", ")}` + ); + const generation = await generateDevotional(candidate.prompt, signal); + console.log("Generated Devotional:\n", generation.message); + return generation; + }, + judge: (candidate, generation) => judgeDraft(candidate, generation, track, holiday, signal), + maxAttempts: JUDGE_MAX_ATTEMPTS, + signal, + onFailedAttempt: (attempt) => recordRejections( + supabase, isoDate, devotionalType, targetTestament, track, + promptVersion, holiday, [attempt], runId, + ), + }); - versesUsed = verseRefs.map((ref) => { - const text = lookupVerseText(ref.book, ref.chapter, ref.verse); - const isOT = OT_BOOKS.some((b) => b.name === ref.book); - return { - book: ref.book, - chapter: ref.chapter, - verse: ref.verse, - text: text || `[${ref.book} ${ref.chapter}:${ref.verse}]`, - testament: isOT ? ("old" as const) : ("new" as const), - }; + const { accepted, rejected, exhausted, retryError } = outcome; + if (exhausted) { + await captureException( + new Error(`daily-devotional judge: ${isoDate}: ${accepted ? "using soft-score fallback" : "no usable draft"}${retryError ? `; ${retryError}` : ""}`), + { functionName: "daily-devotional", tags: { stage: "judge-exhausted", track }, extra: { runId } }, + ); + } + if (!accepted) { + return new Response(JSON.stringify({ error: "No draft passed the safety criteria; existing devotional preserved", runId }), { + status: 503, headers: { "Content-Type": "application/json" }, }); - - prompt = buildMultiPrompt(track, versesUsed, formatted, holiday); - } else { - const verse = selectVerse(targetTestament, holiday); - // Resolve exact text from bible.json for holiday verses - if (holiday) { - const resolvedText = lookupVerseText( - verse.book, - verse.chapter, - verse.verse - ); - if (resolvedText) verse.text = resolvedText; - } - versesUsed = [verse]; - prompt = buildSinglePrompt(track, verse, formatted, holiday); } - - console.log( - `Verses: ${versesUsed.map((v) => `${v.book} ${v.chapter}:${v.verse}`).join(", ")}` - ); - - // Generate devotional - const { message: devotional, usage: generationUsage } = - await generateDevotional(prompt); - console.log("Generated Devotional:\n", devotional); + const { prompt, verseSelectionPrompt, selectionUsage } = accepted.candidate; + const versesUsed = accepted.candidate.verses; + const devotional = accepted.generation.message; const devotionalUsage: DevotionalUsage = { - generation: generationUsage, + generation: accepted.generation.usage, ...(selectionUsage ? { selection: selectionUsage } : {}), + ...(accepted.generation.judgeUsage ? { judge: accepted.generation.judgeUsage } : {}), + }; + + const storedJudge: StoredJudge = { + run_id: runId, + ...(retryError ? { retry_error: retryError } : {}), + model: JUDGE_MODEL, + attempt: accepted.attempt, + attempts: rejected.length + 1, + exhausted, + result: accepted.judge, + ...(accepted.judgeError ? { error: accepted.judgeError } : {}), }; // Theme metadata for the iOS app's "why am I seeing this?" display. @@ -2461,7 +2580,8 @@ Deno.serve(async (req) => { track, verseSelectionPrompt, }, - themeMetadata + themeMetadata, + storedJudge ); return new Response(devotional, { @@ -2475,7 +2595,12 @@ Deno.serve(async (req) => { headers: { "Content-Type": "application/json" }, }); } -}); +} + +if (import.meta.main) { + initSentry("daily-devotional"); + Deno.serve(handleRequest); +} /* To invoke locally: diff --git a/supabase/functions/daily-devotional/judge_eval_test.ts b/supabase/functions/daily-devotional/judge_eval_test.ts new file mode 100644 index 0000000..d59df33 --- /dev/null +++ b/supabase/functions/daily-devotional/judge_eval_test.ts @@ -0,0 +1,121 @@ +// Live evals for the devotional judge: runs the REAL judge model over the +// labelled cases in evals/judge_cases.ts and checks it makes the publish +// decision a careful editor would. +// +// Costs money and needs the network, so it is opt-in and skipped in CI: +// +// RUN_JUDGE_EVALS=1 OPENAI_API_KEY=sk-... \ +// deno test --allow-all supabase/functions/daily-devotional/judge_eval_test.ts +// +// or `make eval-devotional-judge`. Optional: +// JUDGE_EVAL_TRIALS=3 run each case N times (the judge is not +// deterministic; a case must be right on every trial) +// JUDGE_EVAL_CASE=psalm only run cases whose id contains this +// +// Run it whenever the rubric, weights, threshold or JUDGE_MODEL change, and +// bump JUDGE_PROMPT_VERSION when the rubric does. The deterministic checks on +// the fixtures themselves live in judge_test.ts and always run. + +import { + buildJudgePrompt, + callJudgeModel, + type JudgeResult, + parseJudgeResponse, +} from "../_shared/devotional-judge.ts"; +import { type JudgeCase, JUDGE_CASES } from "./evals/judge_cases.ts"; + +const enabled = Deno.env.get("RUN_JUDGE_EVALS") === "1" && + !!Deno.env.get("OPENAI_API_KEY"); + +// The judge model is set in index.ts beside the other model constants (one +// source of truth). index.ts cannot be imported (top-level Deno.serve), so +// read the literal the same way holidays_test.ts reads the others. +function judgeModelFromSource(): string { + const source = Deno.readTextFileSync(new URL("./index.ts", import.meta.url)); + const m = source.match(/const JUDGE_MODEL = "([^"]+)";/); + if (!m) throw new Error('index.ts must declare const JUDGE_MODEL = "...";'); + return m[1]; +} + +interface Trial { + result: JudgeResult | null; + error?: string; +} + +async function judgeOnce(c: JudgeCase, model: string, apiKey: string): Promise { + try { + const { content } = await callJudgeModel({ + apiKey, + model, + prompt: buildJudgePrompt({ + message: c.message, + track: c.track, + verses: c.verses, + holidayName: c.holidayName ?? null, + }), + }); + return { result: parseJudgeResponse(content) }; + } catch (err) { + return { result: null, error: err instanceof Error ? err.message : String(err) }; + } +} + +function trialProblem(c: JudgeCase, t: Trial): string | null { + if (!t.result) return `judge error: ${t.error}`; + const decision = t.result.passed ? "pass" : "fail"; + if (decision !== c.expect) return `expected ${c.expect}, judge said ${decision}`; + if (c.hardFails && !c.hardFails.some((h) => t.result!.hard_fails.includes(h))) { + return `failed, but without any of the expected hard fails [${c.hardFails.join(", ")}]`; + } + return null; +} + +Deno.test({ + name: "devotional judge agrees with labelled cases (live model)", + ignore: !enabled, + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const apiKey = Deno.env.get("OPENAI_API_KEY")!; + const model = judgeModelFromSource(); + const trials = Math.max(1, Number(Deno.env.get("JUDGE_EVAL_TRIALS") ?? "1")); + const filter = Deno.env.get("JUDGE_EVAL_CASE"); + const cases = JUDGE_CASES.filter((c) => !filter || c.id.includes(filter)); + if (cases.length === 0) throw new Error(`no cases match JUDGE_EVAL_CASE=${filter}`); + + const results = await Promise.all( + cases.map(async (c) => ({ + c, + trials: await Promise.all( + Array.from({ length: trials }, () => judgeOnce(c, model, apiKey)), + ), + })), + ); + + const failures: string[] = []; + console.log(`\njudge model: ${model}, ${cases.length} cases x ${trials} trial(s)\n`); + for (const { c, trials: ts } of results) { + const problems = ts.map((t) => trialProblem(c, t)); + const ok = problems.every((p) => p === null); + const scores = ts.map((t) => + t.result + ? `${t.result.overall.toFixed(1)}${t.result.hard_fails.length ? `[${t.result.hard_fails.join(",")}]` : ""}` + : "ERR" + ).join(" "); + console.log(`${ok ? "ok " : "MISS"} ${c.expect.padEnd(4)} ${scores.padEnd(28)} ${c.id}`); + if (!ok) { + const first = ts.find((_t, i) => problems[i] !== null); + failures.push( + `${c.id}: ${problems.filter(Boolean).join("; ")}\n label: ${c.why}\n judge: ${ + first?.result ? JSON.stringify(first.result.scores) + " " + first.result.reasons.join(" | ") : first?.error + }`, + ); + } + } + const correct = results.length - failures.length; + console.log(`\n${correct}/${results.length} cases correct\n`); + if (failures.length > 0) { + throw new Error(`judge disagreed with ${failures.length} labelled case(s):\n ${failures.join("\n ")}`); + } + }, +}); diff --git a/supabase/functions/daily-devotional/judge_recovery_test.ts b/supabase/functions/daily-devotional/judge_recovery_test.ts new file mode 100644 index 0000000..3481fae --- /dev/null +++ b/supabase/functions/daily-devotional/judge_recovery_test.ts @@ -0,0 +1,136 @@ +import { + generateWithJudge, + JUDGE_CRITERIA, + type JudgeResult, + parseJudgeResponse, +} from "../_shared/devotional-judge.ts"; + +function verdict(score: number, hard = false): JudgeResult { + return parseJudgeResponse(JSON.stringify({ + scores: Object.fromEntries( + JUDGE_CRITERIA.map((c) => [c, hard && c === "faithfulness" ? 2 : score]), + ), + hard_fails: hard ? ["misquoted_scripture"] : [], + reasons: [], + })); +} +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(message); +} + +for (const stage of ["selection", "generation"]) { + Deno.test(`a later ${stage} failure retains the earlier usable draft`, async () => { + const outcome = await generateWithJudge({ + first: 1, + next: () => + stage === "selection" + ? Promise.reject(new Error("selection failed")) + : Promise.resolve(2), + generate: (n) => + n === 2 + ? Promise.reject(new Error("generation failed")) + : Promise.resolve("draft"), + judge: () => Promise.resolve(verdict(3)), + log: () => {}, + }); + assert( + outcome.accepted?.candidate === 1, + "the completed draft must survive a retry error", + ); + assert(outcome.exhausted, "fallback should be identified"); + }); +} + +Deno.test("exhaustion excludes hard failures even when they score higher", async () => { + const outcome = await generateWithJudge({ + first: 1, + next: () => Promise.resolve(2), + generate: () => Promise.resolve("draft"), + judge: (n) => Promise.resolve(verdict(n === 1 ? 3 : 5, n === 2)), + maxAttempts: 2, + log: () => {}, + }); + assert( + outcome.accepted?.candidate === 1, + "known misquotes must not win the fallback ranking", + ); +}); + +Deno.test("when every draft hard-fails nothing is approved for publication", async () => { + const outcome = await generateWithJudge({ + first: 1, + next: () => Promise.resolve(2), + generate: () => Promise.resolve("draft"), + judge: () => Promise.resolve(verdict(5, true)), + maxAttempts: 2, + log: () => {}, + }); + assert( + outcome.accepted === null, + "all-hard-failure outcome must preserve the existing devotional", + ); + assert( + outcome.rejected.length === 2, + "both failures remain available for auditing", + ); +}); + +Deno.test("failed drafts are audited before a retry and survive deadline exhaustion", async () => { + const controller = new AbortController(); + const events: string[] = []; + const outcome = await generateWithJudge({ + first: 1, + next: () => { + events.push("next"); + return Promise.resolve(2); + }, + generate: () => Promise.resolve("draft"), + judge: () => Promise.resolve(verdict(3)), + onFailedAttempt: () => { + events.push("audit"); + controller.abort(new Error("shared time budget expired")); + return Promise.resolve(); + }, + signal: controller.signal, + log: () => {}, + }); + assert( + events.join() === "audit", + "no new model work should start after the shared deadline", + ); + assert( + outcome.accepted?.candidate === 1, + "completed usable draft must survive a deadline", + ); + assert( + outcome.retryError?.includes("time budget"), + "deadline reason must be retained", + ); +}); + +Deno.test("an audit failure does not discard a draft or block a later passing draft", async () => { + const outcome = await generateWithJudge({ + first: 1, + next: () => Promise.resolve(2), + generate: () => Promise.resolve("draft"), + judge: (n) => Promise.resolve(verdict(n === 1 ? 3 : 4)), + onFailedAttempt: () => Promise.reject(new Error("audit down")), + log: () => {}, + }); + assert( + outcome.accepted?.candidate === 2, + "publishing should remain available when the audit store fails", + ); +}); + +Deno.test("a floor-level criterion cannot be used as a soft-score fallback", async () => { + const outcome = await generateWithJudge({ + first: 1, + next: () => Promise.resolve(2), + generate: () => Promise.resolve("draft"), + judge: () => Promise.resolve(verdict(2)), + maxAttempts: 1, + log: () => {}, + }); + assert(outcome.accepted === null, "floor-level drafts must not be published"); +}); diff --git a/supabase/functions/daily-devotional/judge_test.ts b/supabase/functions/daily-devotional/judge_test.ts new file mode 100644 index 0000000..007642f --- /dev/null +++ b/supabase/functions/daily-devotional/judge_test.ts @@ -0,0 +1,318 @@ +// deno-lint-ignore-file require-await -- stubbed model calls stand in for async APIs +// Deterministic tests for the devotional quality gate. No network, no model: +// the parser, the pass/fail rule, the retry loop (driven by stubs), the +// index.ts wiring, and the eval fixtures themselves. The live-model evals are +// judge_eval_test.ts (opt-in). + +import { + buildJudgePrompt, + callJudgeModel, + generateWithJudge, + JUDGE_CRITERIA, + JUDGE_HARD_FAILS, + JUDGE_PASS_THRESHOLD, + type JudgeResult, + overallScore, + parseJudgeResponse, + TRACK_INTENT, +} from "../_shared/devotional-judge.ts"; +import { JUDGE_CASES } from "./evals/judge_cases.ts"; +import bibleJson from "./bible.json" with { type: "json" }; + +function assert(cond: unknown, msg: string): asserts cond { + if (!cond) throw new Error(msg); +} + +function judgeJson( + scores: Partial>, + hard_fails: string[] = [], + reasons: string[] = [], +): string { + const full = Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, scores[c] ?? 4])); + return JSON.stringify({ scores: full, hard_fails, reasons }); +} + +const indexSource = () => Deno.readTextFileSync(new URL("./index.ts", import.meta.url)); + +// ─── Scoring ──────────────────────────────────────────────────────── + +Deno.test("overall maps all-1 to 0 and all-5 to 10", () => { + const all = (n: number) => Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, n])) as never; + assert(overallScore(all(1)) === 0, `all-1 → ${overallScore(all(1))}`); + assert(overallScore(all(5)) === 10, `all-5 → ${overallScore(all(5))}`); + assert(overallScore(all(3)) === 5, `all-3 → ${overallScore(all(3))}`); +}); + +Deno.test("all-4s passes, all-3s fails", () => { + assert(parseJudgeResponse(judgeJson({})).passed, "all 4s should pass"); + const threes = parseJudgeResponse( + judgeJson(Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, 3]))), + ); + assert(!threes.passed, `all 3s should fail (overall ${threes.overall})`); +}); + +Deno.test("a single criterion at the floor fails even with a high average", () => { + // The Numbers 26:58 shape: well-formed, on-track, but the verse can't carry it. + const r = parseJudgeResponse(judgeJson({ verse_weight: 1, faithfulness: 5, substance: 5, track_fit: 5, craft: 5 })); + assert(r.overall >= JUDGE_PASS_THRESHOLD, `precondition: overall ${r.overall} should clear threshold`); + assert(!r.passed, "a floor-level criterion must fail the draft"); +}); + +Deno.test("any hard fail fails a perfect score", () => { + const all5 = Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, 5])); + const r = parseJudgeResponse(judgeJson(all5, ["misquoted_scripture"])); + assert(!r.passed, "hard fail must fail"); + assert(r.hard_fails.length === 1, "hard fail kept"); +}); + +Deno.test("verdict comes from scores, not from a verdict the model writes", () => { + const ones = Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, 1])); + const r = parseJudgeResponse(JSON.stringify({ scores: ones, hard_fails: [], passed: true, verdict: "pass" })); + assert(!r.passed, "model-written verdict must be ignored"); +}); + +Deno.test("parser clamps, rounds, and drops unknown hard fails", () => { + const r = parseJudgeResponse( + judgeJson({ verse_weight: 9, faithfulness: 0, substance: 3.6 }, ["made_up", "broken_format", "broken_format"]), + ); + assert(r.scores.verse_weight === 5, "clamped high"); + assert(r.scores.faithfulness === 1, "clamped low"); + assert(r.scores.substance === 4, "rounded"); + assert(JSON.stringify(r.hard_fails) === '["broken_format"]', `hard fails: ${r.hard_fails}`); +}); + +Deno.test("parser accepts numeric strings and fenced JSON", () => { + const body = judgeJson({}).replace('"craft":4', '"craft":"5"'); + const r = parseJudgeResponse("```json\n" + body + "\n```"); + assert(r.scores.craft === 5, "string score parsed"); +}); + +Deno.test("parser throws on missing scores so the caller fails open", () => { + for (const bad of ["not json", "{}", JSON.stringify({ scores: { verse_weight: 4 } })]) { + let threw = false; + try { + parseJudgeResponse(bad); + } catch { + threw = true; + } + assert(threw, `should throw on ${bad}`); + } +}); + +// ─── Prompt ───────────────────────────────────────────────────────── + +Deno.test("judge prompt carries the verse, the track's intent, and the draft", () => { + const p = buildJudgePrompt({ + message: "DRAFT-BODY", + track: "lament", + verses: [{ book: "Psalms", chapter: 13, verse: 1, text: "How long wilt thou forget me, O LORD?" }], + holidayName: "Good Friday", + }); + for (const needle of ["DRAFT-BODY", "Psalms 13:1", "How long wilt thou forget me", "Deliberately does NOT resolve", "Good Friday", ...JUDGE_CRITERIA, ...JUDGE_HARD_FAILS]) { + assert(p.includes(needle), `judge prompt missing: ${needle}`); + } +}); + +Deno.test("every track in TRACK_CYCLE has a judge intent", () => { + const m = indexSource().match(/const TRACK_CYCLE = \[([\s\S]*?)\] as const;/); + assert(m, "TRACK_CYCLE not found"); + const tracks = [...m[1].matchAll(/"([^"]+)"/g)].map((x) => x[1]); + assert(tracks.length > 0, "no tracks parsed"); + for (const t of tracks) assert(TRACK_INTENT[t], `TRACK_INTENT missing "${t}" — the judge would grade it generically`); +}); + +// ─── Model call ───────────────────────────────────────────────────── + +Deno.test("callJudgeModel requests JSON from the given model", async () => { + let sent: Record = {}; + const fakeFetch = (async (_url: string, init: RequestInit) => { + sent = JSON.parse(init.body as string); + return new Response(JSON.stringify({ choices: [{ message: { content: judgeJson({}) } }], usage: { prompt_tokens: 10 } })); + }) as unknown as typeof fetch; + const { content, data } = await callJudgeModel({ apiKey: "k", model: "judge-x", prompt: "p", fetchFn: fakeFetch }); + assert(sent.model === "judge-x", "model passed through"); + assert((sent.response_format as { type: string }).type === "json_object", "json mode"); + assert(parseJudgeResponse(content).passed, "content returned"); + assert((data as { usage: { prompt_tokens: number } }).usage.prompt_tokens === 10, "raw data returned for costing"); +}); + +Deno.test("callJudgeModel throws on HTTP error and empty content", async () => { + for (const res of [new Response("boom", { status: 500 }), new Response(JSON.stringify({ choices: [] }))]) { + const fakeFetch = (async () => res) as unknown as typeof fetch; + let threw = false; + try { + await callJudgeModel({ apiKey: "k", model: "m", prompt: "p", fetchFn: fakeFetch }); + } catch { + threw = true; + } + assert(threw, `should throw for status ${res.status}`); + } +}); + +// ─── Retry loop ───────────────────────────────────────────────────── + +const pass = (overall = 8): JudgeResult => ({ + version: "t", scores: Object.fromEntries(JUDGE_CRITERIA.map((c) => [c, 4])) as JudgeResult["scores"], overall, threshold: 7, hard_fails: [], passed: true, reasons: [], +}); +const fail = (overall: number): JudgeResult => ({ ...pass(overall), passed: false }); + +// Drives generateWithJudge with verse-name candidates and a scripted judge. +async function run(script: Array, maxAttempts = 3) { + const verses = ["Numbers 26:58", "Psalms 13:1", "John 11:35", "Ruth 1:16"]; + const nextCalls: string[][] = []; + let i = 0; + const outcome = await generateWithJudge({ + first: verses[0], + next: async (tried) => { + nextCalls.push([...tried]); + return verses[tried.length]; + }, + generate: async (v) => ({ message: `devotional on ${v}` }), + judge: async () => { + const r = script[i++]; + if (r instanceof Error) throw r; + return r; + }, + maxAttempts, + log: () => {}, + }); + assert(outcome.accepted, "script should produce a usable draft"); + return { outcome: { ...outcome, accepted: outcome.accepted }, nextCalls, judgeCalls: i }; +} + +Deno.test("first draft passes: published, nothing rejected, no retry", async () => { + const { outcome, nextCalls } = await run([pass()]); + assert(outcome.accepted.candidate === "Numbers 26:58", "first verse kept"); + assert(outcome.rejected.length === 0 && !outcome.exhausted, "nothing rejected"); + assert(nextCalls.length === 0, "no new verse requested"); +}); + +Deno.test("failed draft is rejected and a different verse is tried", async () => { + const { outcome, nextCalls } = await run([fail(4), pass()]); + assert(outcome.accepted.candidate === "Psalms 13:1", `accepted ${outcome.accepted.candidate}`); + assert(outcome.accepted.attempt === 2, "attempt number"); + assert(outcome.rejected.length === 1, "one rejection"); + assert(outcome.rejected[0].candidate === "Numbers 26:58", "the failed verse is the rejected one"); + assert(outcome.rejected[0].generation.message.includes("Numbers 26:58"), "rejected draft kept for the record"); + assert(JSON.stringify(nextCalls) === '[["Numbers 26:58"]]', `next() sees what was tried: ${JSON.stringify(nextCalls)}`); + assert(!outcome.exhausted, "not exhausted"); +}); + +Deno.test("stops at maxAttempts and publishes the best-scoring failure", async () => { + const { outcome, judgeCalls } = await run([fail(3), fail(6.5), fail(5)]); + assert(judgeCalls === 3, `judged ${judgeCalls} times`); + assert(outcome.exhausted, "exhausted"); + assert(outcome.accepted.candidate === "Psalms 13:1", `best was attempt 2, got ${outcome.accepted.candidate}`); + assert(outcome.rejected.length === 2, "the other two are rejected"); + assert(!outcome.rejected.includes(outcome.accepted), "published draft is never also recorded as rejected"); +}); + +Deno.test("judge outage fails open: the draft is published unjudged", async () => { + const { outcome, nextCalls } = await run([new Error("503")]); + assert(outcome.accepted.judge === null, "no judge result"); + assert(outcome.accepted.judgeError === "503", "error kept for the row"); + assert(outcome.rejected.length === 0 && nextCalls.length === 0, "no retry on outage"); +}); + +Deno.test("judge outage after a real fail keeps the earlier rejection", async () => { + const { outcome } = await run([fail(4), new Error("timeout")]); + assert(outcome.accepted.attempt === 2 && outcome.accepted.judge === null, "second draft published unjudged"); + assert(outcome.rejected.length === 1, "first draft still rejected"); +}); + +Deno.test("generation errors propagate (handler returns 500 as before)", async () => { + let threw = false; + try { + await generateWithJudge({ + first: "a", + next: async () => "b", + generate: async () => { + throw new Error("openai down"); + }, + judge: async () => pass(), + log: () => {}, + }); + } catch (e) { + threw = (e as Error).message === "openai down"; + } + assert(threw, "generation error must propagate"); +}); + +// ─── index.ts wiring (source checks; index.ts can't be imported) ──── + +Deno.test("JUDGE_MODEL is a single literal with no env fallback, priced", () => { + const source = indexSource(); + const m = source.match(/const JUDGE_MODEL = "([^"]+)";/); + assert(m, 'JUDGE_MODEL must be a single string literal: const JUDGE_MODEL = "...";'); + assert(!/JUDGE_MODEL\s*=\s*[\s\S]{0,40}Deno\.env\.get/.test(source), "JUDGE_MODEL must not read from env"); + assert(source.includes(`"${m[1]}":`), `MODEL_PRICING has no entry for ${m[1]}; judge cost would record as 0`); +}); + +Deno.test("handler gates generation through the judge and records rejections", () => { + const source = indexSource(); + for (const marker of [ + "generateWithJudge", + "judgeDraft(candidate, generation, track, holiday, signal)", + "onFailedAttempt: (attempt) => recordRejections(", + '.from("daily_devotional_rejections").insert(', + "judge,\n", // saveDevotional persists the judge column + ]) { + assert(source.includes(marker), `index.ts missing: ${JSON.stringify(marker)}`); + } + // Rejections must be written before the published row, so a failed save + // can't lose the record of why earlier drafts were thrown away. + assert( + source.indexOf("onFailedAttempt: (attempt) => recordRejections(") < source.indexOf("await saveDevotional("), + "recordRejections must run before saveDevotional", + ); +}); + +Deno.test("retries keep track, type and testament; only the verse changes", () => { + const source = indexSource(); + assert( + source.includes("buildCandidate(devotionalType, targetTestament, track, holiday, formatted, tried, signal)"), + "retry candidates must be built from the same devotionalType/targetTestament/track", + ); +}); + +Deno.test("migration creates the rejections table locked to service_role", () => { + const dir = new URL("../../migrations/", import.meta.url); + const file = [...Deno.readDirSync(dir)].map((e) => e.name).find((n) => n.endsWith("_add_devotional_judge.sql")); + assert(file, "judge migration not found"); + const sql = Deno.readTextFileSync(new URL(file, dir)); + for (const needle of [ + 'ADD COLUMN IF NOT EXISTS judge JSONB', + "CREATE TABLE IF NOT EXISTS public.daily_devotional_rejections", + "ENABLE ROW LEVEL SECURITY", + "REVOKE ALL ON TABLE public.daily_devotional_rejections FROM anon", + ]) { + assert(sql.includes(needle), `migration missing: ${needle}`); + } +}); + +// ─── Eval fixtures ────────────────────────────────────────────────── + +Deno.test("eval cases: unique ids, both labels present, known tracks", () => { + const ids = new Set(JUDGE_CASES.map((c) => c.id)); + assert(ids.size === JUDGE_CASES.length, "duplicate case id"); + assert(JUDGE_CASES.some((c) => c.expect === "pass"), "need at least one pass case"); + assert(JUDGE_CASES.some((c) => c.expect === "fail"), "need at least one fail case"); + for (const c of JUDGE_CASES) assert(TRACK_INTENT[c.track], `${c.id}: unknown track ${c.track}`); +}); + +Deno.test("eval cases quote exact KJV text", () => { + // A fixture typo would teach the judge to flag misquotes that aren't there. + const books = bibleJson as unknown as Array<{ + name: string; + chapters: Array<{ number: number; paragraphs: Array<{ startingVerse: number; text: string }> }>; + }>; + for (const c of JUDGE_CASES) { + for (const v of c.verses) { + const para = books.find((b) => b.name === v.book)?.chapters.find((ch) => ch.number === v.chapter) + ?.paragraphs.find((p) => p.startingVerse === v.verse); + assert(para, `${c.id}: ${v.book} ${v.chapter}:${v.verse} not a paragraph start in bible.json`); + const kjv = para.text.replace(/<\/?JESUS>/g, "").trim(); + assert(kjv === v.text, `${c.id}: verse text differs from bible.json\n fixture: ${v.text}\n kjv: ${kjv}`); + } + } +}); diff --git a/supabase/functions/daily-devotional/verse_selection_test.ts b/supabase/functions/daily-devotional/verse_selection_test.ts index 0287236..19e49bd 100644 --- a/supabase/functions/daily-devotional/verse_selection_test.ts +++ b/supabase/functions/daily-devotional/verse_selection_test.ts @@ -10,6 +10,8 @@ import { MAX_PARAGRAPH_CHARS, MIN_PARAGRAPH_CHARS, selectRandomVerse, + UNSUITABLE_PATTERNS, + unsuitableReason, type VerseBook, } from "../_shared/verse-selection.ts"; import bibleJson from "./bible.json" with { type: "json" }; @@ -106,3 +108,65 @@ Deno.test("sampling the real function tracks the pool distribution", () => { ); } }); + +// ─── Unsuitable passage shapes ────────────────────────────────────── + +const inPool = (pool: ReturnType, book: string, chapter: number, verse: number) => + pool.some((v) => v.book === book && v.chapter === chapter && v.verse === verse); + +Deno.test("census, itinerary, begat and measurement passages never reach the pool", () => { + const ot = eligibleParagraphs(OT, "old"); + const nt = eligibleParagraphs(NT, "new"); + const blocked: Array<[typeof ot, string, number, number, string]> = [ + [ot, "Numbers", 26, 58, "clan-list"], // 2026-10-11, the day that prompted this + [ot, "Numbers", 33, 19, "itinerary"], + [ot, "Genesis", 5, 18, "begat-chain"], + [ot, "Exodus", 26, 16, "measurements"], + [nt, "Acts", 7, 8, "begat-chain"], + ]; + for (const [pool, book, ch, v, rule] of blocked) { + if (inPool(pool, book, ch, v)) throw new Error(`${book} ${ch}:${v} (${rule}) should be filtered out`); + } +}); + +Deno.test("passages a broader name/measure rule wrongly caught stay in the pool", () => { + // The rejected "too many proper names" rule took every one of these, and + // "talents" as a measurement took the parable. Keep the rules narrow. + const nt = eligibleParagraphs(NT, "new"); + const kept: Array<[string, number, number]> = [ + ["John", 19, 19], // the title on the cross + ["Mark", 15, 47], // Mary Magdalene sees where he was laid + ["Matthew", 10, 2], // the twelve named + ["Matthew", 25, 20], // parable of the talents + ["John", 1, 40], // Andrew follows + ]; + for (const [book, ch, v] of kept) { + if (!inPool(nt, book, ch, v)) throw new Error(`${book} ${ch}:${v} should stay eligible`); + } + const ot = eligibleParagraphs(OT, "old"); + for (const [book, ch, v] of [["Psalms", 23, 1], ["Isaiah", 40, 30], ["Ruth", 1, 16]] as const) { + if (!inPool(ot, book, ch, v)) throw new Error(`${book} ${ch}:${v} should stay eligible`); + } +}); + +Deno.test("unsuitable rules stay narrow: under 1% of the pool, almost all OT", () => { + // Measured at introduction: ~177 of ~24,000, one NT. A jump here means a + // rule got broader and is now eating real devotional material. + let ot = 0, nt = 0, total = 0; + for (const [books, testament] of [[OT, "old"], [NT, "new"]] as const) { + for (const b of books) for (const c of b.chapters) for (const p of c.paragraphs) { + if (p.text.length > MAX_PARAGRAPH_CHARS || p.text.trim().length < MIN_PARAGRAPH_CHARS) continue; + total++; + if (unsuitableReason(p.text.replace(/<\/?JESUS>/g, ""))) testament === "old" ? ot++ : nt++; + } + } + if ((ot + nt) / total > 0.01) throw new Error(`rules drop ${ot + nt}/${total} (>1%)`); + if (nt > 3) throw new Error(`rules drop ${nt} NT paragraphs; expected ~1`); +}); + +Deno.test("each unsuitable rule has a sane minimum", () => { + for (const r of UNSUITABLE_PATTERNS) { + if (!r.pattern.global) throw new Error(`${r.name}: pattern must be /g so matches are counted`); + if (r.min < 1) throw new Error(`${r.name}: min must be >= 1`); + } +}); diff --git a/supabase/migrations/20261011120000_add_devotional_judge.sql b/supabase/migrations/20261011120000_add_devotional_judge.sql new file mode 100644 index 0000000..d0d199f --- /dev/null +++ b/supabase/migrations/20261011120000_add_devotional_judge.sql @@ -0,0 +1,54 @@ +-- Quality judgments for generated daily devotionals. +-- Every failed judgment is audited before retrying. A soft-score fallback +-- may subsequently be published; hard failures and floor-level criteria may +-- not. run_id + attempt link an audit to Daily Devotional.judge metadata. +-- Published metadata contains model, run_id, attempt, attempts, exhausted, +-- result, and optional error (judge unavailable) or retry_error. +-- NULL judge means a custom row or content predating the quality gate. + +ALTER TABLE public."Daily Devotional" + ADD COLUMN IF NOT EXISTS judge JSONB; + +COMMENT ON COLUMN public."Daily Devotional".judge IS + 'Judge result for the published draft: {model, run_id, attempt, attempts, exhausted, result, error?, retry_error?}. NULL for custom devotionals or rows predating the judge.'; + +CREATE TABLE IF NOT EXISTS public.daily_devotional_rejections ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + run_id UUID NOT NULL, + for_date DATE NOT NULL, + attempt INT NOT NULL, + track TEXT, + prompt_version TEXT, + devotional_type TEXT, + testament TEXT, + verses JSONB, + holiday_name TEXT, + message TEXT NOT NULL, + prompt TEXT, + verse_selection_prompt TEXT, + model TEXT, + usage JSONB, + judge_model TEXT, + judge_version TEXT, + judge_score NUMERIC(4, 1), + judge_threshold NUMERIC(4, 1), + judge JSONB, + UNIQUE (run_id, attempt) +); + +COMMENT ON TABLE public.daily_devotional_rejections IS + 'Every failed AI judgment, persisted before retrying. A soft-score fallback may also be published; match run_id and attempt to the published judge metadata. Never read by the apps.'; + +CREATE INDEX IF NOT EXISTS daily_devotional_rejections_for_date_idx + ON public.daily_devotional_rejections (for_date); +CREATE INDEX IF NOT EXISTS daily_devotional_rejections_track_idx + ON public.daily_devotional_rejections (track); + +-- Server-side only, same posture as "Daily Devotional" after +-- 20260529162804_lock_daily_devotional_direct_reads.sql: RLS on with no +-- policies, no anon/authenticated grants. +ALTER TABLE public.daily_devotional_rejections ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON TABLE public.daily_devotional_rejections FROM anon; +REVOKE ALL ON TABLE public.daily_devotional_rejections FROM authenticated; +GRANT ALL ON TABLE public.daily_devotional_rejections TO service_role;