From afc9e016813211ee0af9b9d6e44421a966cfd2e3 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Sun, 11 Oct 2026 03:53:09 +0300 Subject: [PATCH] Show campaign A/B results and weighted variations in human output campaign analytics renders the A/B criterion, outcome, holdout and reserve counts and a per-variant table with the winner marked. campaign view (and create) show the variant count and outcome, and the number of weighted message alternatives. campaign create help and the README describe the variants, variantStrategy and messageVariations beta input fields. The published SDK pin types these fields as unknown or omits them, so the CLI narrows them at runtime with local types that accept the SDK main types from polymorfa/sdks#344. --- README.md | 63 ++++ src/commands/campaign/create.ts | 13 +- src/services/campaign-experiments.ts | 257 ++++++++++++++++ src/services/campaigns.ts | 7 + test/commands/campaign-experiments.test.ts | 327 +++++++++++++++++++++ 5 files changed, 666 insertions(+), 1 deletion(-) create mode 100644 src/services/campaign-experiments.ts create mode 100644 test/commands/campaign-experiments.test.ts diff --git a/README.md b/README.md index da726c3c..e36af8da 100644 --- a/README.md +++ b/README.md @@ -1714,6 +1714,69 @@ Requeue also accepts the header, but the API does not deduplicate it. Each invocation moves recipients eligible at that time, so a retry can move newly failed recipients rather than returning the first result. +### A/B tests and weighted message variation (beta) + +Both are betas. `campaign create` sends the `--input` object as the request +body, so you set them with these fields. A team that is not enrolled gets +`403 feature_unavailable`. There is no campaign update command; create the +campaign with the fields you need. + +- **A/B test:** `variants` holds two to four messages, each with a `key`, a + `label`, an integer `weight` (the weights add up to 100) and a `blueprint`. + `variantStrategy` sets `winnerCriterion` (`delivery`, `read` or `reply`), + `holdoutPercent` (0 to 30), an optional `testSlicePercent` (5 to 50, + default 20), `autoPromote: true` and `testWindowMinutes` (15 to 1,440). An + A/B test cannot use `templateId`. +- **Weighted message variation:** `messageVariations` holds two to five + alternatives, each with a `key`, an integer `weight` from 1 to 99 (the + weights add up to 100) and a `blueprint`. Each recipient receives one + alternative and keeps it across retries. Every alternative must use the same + `{{variable}}` placeholders. Alternatives send only from linked-device + numbers and cannot be combined with `templateId`, composed messages or an + A/B test. + +```json +{ + "name": "October offer", + "variants": [ + { + "key": "a", + "label": "Short", + "weight": 50, + "blueprint": {"version": 2, "source": "Hi {{first_name | there}}!"} + }, + { + "key": "b", + "label": "Long", + "weight": 50, + "blueprint": { + "version": 2, + "source": "Hello {{first_name | there}}, your offer is ready." + } + } + ], + "variantStrategy": { + "winnerCriterion": "read", + "holdoutPercent": 5, + "autoPromote": true, + "testWindowMinutes": 120 + } +} +``` + +At launch, recipients are split into an optional holdout, a test group that +receives the variants and a reserve that waits for the winner. When the test +window ends, the winner is sent to the reserve. If no variant is clearly ahead, +the outcome is inconclusive and the campaign pauses with the reserve unsent. + +`campaign view` shows the variant count and outcome of an A/B test, and the +number of weighted alternatives. `campaign analytics` adds the criterion, the +outcome (the winner, inconclusive with its reason, or no winner yet), the +holdout and reserve counts, and a row per variant with the criterion rate, +sent, assigned, delivered, read and replied counts. The winner row is marked +`(winner)`. `--json` returns the API's `variants`, `variantStrategy`, +`experimentOutcome`, `messageVariations` and `experiment` fields unchanged. + ### Campaign recipients Recipients are appended to a campaign that has not started sending, and read diff --git a/src/commands/campaign/create.ts b/src/commands/campaign/create.ts index be812749..dfa8806f 100644 --- a/src/commands/campaign/create.ts +++ b/src/commands/campaign/create.ts @@ -6,7 +6,18 @@ import {idempotencyKey} from '../../input/idempotency.js' import {createCampaign, renderCampaign} from '../../services/campaigns.js' export default class CampaignCreate extends BaseCommand { - static override description = 'Create a campaign from one JSON object.' + static override description = `Create a campaign from one JSON object. + +The object is sent as the request body. A/B tests and weighted message variation are betas that need team enrollment; a team that is not enrolled gets 403 feature_unavailable: + +- A/B test: "variants" (two to four, each with key, label, integer weight and blueprint; weights add up to 100) and "variantStrategy" (winnerCriterion delivery, read or reply; holdoutPercent; optional testSlicePercent; autoPromote true; testWindowMinutes). View the results with polymorfa campaign analytics. +- Weighted message variation: "messageVariations", two to five alternatives (key, integer weight, blueprint; weights add up to 100). Each recipient receives one alternative. Linked-device numbers only. + +An A/B test cannot use templateId. messageVariations cannot be combined with templateId, composed messages or an A/B test.` + + static override examples = [ + '<%= config.bin %> <%= command.id %> --project-slug launch-project --input campaign.json', + ] static override flags = { 'idempotency-key': Flags.string({ diff --git a/src/services/campaign-experiments.ts b/src/services/campaign-experiments.ts new file mode 100644 index 00000000..5dc19ae6 --- /dev/null +++ b/src/services/campaign-experiments.ts @@ -0,0 +1,257 @@ +import {terminalText} from '../developer/render.js' + +/* + * Campaign A/B tests and weighted message variation (beta). + * + * The published SDK pin types these fields as unknown or omits them, so the + * CLI reads them as unknown values and narrows them here. The shapes follow + * the SDK's CampaignExperimentResults, CampaignExperimentOutcome, + * CampaignVariant and CampaignMessageVariation types. A value that does not + * match is reported as unreadable instead of being printed partially. + */ + +export type ExperimentOutcome = + | {readonly state: 'promoted'; readonly winnerKey: string} + | {readonly reason: string; readonly state: 'inconclusive'} + +export interface ExperimentVariantResult { + readonly assigned: number + readonly delivered: number + readonly key: string + readonly label: string + readonly outcomeRate: number + readonly read: number + readonly replied: number + readonly sent: number + readonly weight: number +} + +export interface ExperimentResults { + readonly criterion: string + readonly holdoutCount: number + readonly outcome: ExperimentOutcome | null + readonly reserveCount: number + readonly variants: readonly ExperimentVariantResult[] +} + +interface VariantSummary { + readonly key: string + readonly label: string +} + +const UNREADABLE = + 'Unreadable in this CLI version; use --json for the full response.' + +const CRITERION_RATE: Readonly> = { + delivery: 'Delivery rate', + read: 'Read rate', + reply: 'Reply rate', +} + +/** Human lines for `campaign analytics`; empty when the campaign is not an A/B test. */ +export function renderExperimentAnalytics(analytics: object): string[] { + const value = field(analytics, 'experiment') + if (value === undefined || value === null) return [] + const experiment = experimentResults(value) + if (experiment === undefined) return ['', `A/B test\t${UNREADABLE}`] + + const rateLabel = + CRITERION_RATE[experiment.criterion] ?? + `${terminalText(experiment.criterion)} rate` + const winnerKey = + experiment.outcome?.state === 'promoted' + ? experiment.outcome.winnerKey + : undefined + + return [ + '', + `A/B criterion\t${rateLabel}`, + `A/B outcome\t${describeOutcome(experiment.outcome, experiment.variants)}`, + `Holdout\t${experiment.holdoutCount}`, + `Reserve\t${experiment.reserveCount}`, + '', + [ + 'Variant', + rateLabel, + 'Sent', + 'Assigned', + 'Delivered', + 'Read', + 'Replied', + ].join('\t'), + ...experiment.variants.map((variant) => + [ + `${variantName(variant)}${variant.key === winnerKey ? ' (winner)' : ''}`, + percent(variant.outcomeRate), + variant.sent, + variant.assigned, + variant.delivered, + variant.read, + variant.replied, + ].join('\t'), + ), + ] +} + +/** Human lines for `campaign view` covering A/B variants and weighted alternatives. */ +export function renderCampaignBetas(campaign: object): string[] { + const lines: string[] = [] + + const variants = field(campaign, 'variants') + if (variants !== undefined && variants !== null) { + const summaries = variantSummaries(variants) + const outcome = field(campaign, 'experimentOutcome') + const parsed = + outcome === undefined || outcome === null + ? null + : experimentOutcome(outcome) + if (summaries === undefined || parsed === undefined) { + lines.push(`A/B test\t${UNREADABLE}`) + } else { + lines.push( + `A/B test\t${summaries.length} ${summaries.length === 1 ? 'variant' : 'variants'}`, + `A/B outcome\t${describeOutcome(parsed, summaries)}`, + ) + } + } + + const variations = field(campaign, 'messageVariations') + if (variations !== undefined && variations !== null) { + lines.push( + Array.isArray(variations) + ? `Message variations\t${variations.length} weighted ${variations.length === 1 ? 'alternative' : 'alternatives'}` + : `Message variations\t${UNREADABLE}`, + ) + } + + return lines +} + +function describeOutcome( + outcome: ExperimentOutcome | null, + variants: readonly VariantSummary[], +): string { + if (outcome === null) return 'No winner yet' + if (outcome.state === 'promoted') { + const winner = variants.find((variant) => variant.key === outcome.winnerKey) + return `Winner: ${winner === undefined ? terminalText(outcome.winnerKey) : variantName(winner)}, promoted to the reserve` + } + return `Inconclusive (${terminalText(outcome.reason)}); the reserve was not sent` +} + +function variantName(variant: VariantSummary): string { + return `${terminalText(variant.label)} [${terminalText(variant.key)}]` +} + +function percent(rate: number): string { + return `${(Math.round(rate * 1000) / 10).toFixed(1)}%` +} + +function experimentResults(value: unknown): ExperimentResults | undefined { + if (!isRecord(value)) return undefined + const {criterion, holdoutCount, reserveCount, variants} = value + const outcome = + value.outcome === null || value.outcome === undefined + ? null + : experimentOutcome(value.outcome) + if ( + typeof criterion !== 'string' || + !isCount(holdoutCount) || + !isCount(reserveCount) || + outcome === undefined || + !Array.isArray(variants) + ) { + return undefined + } + const results: ExperimentVariantResult[] = [] + for (const variant of variants) { + const parsed = variantResult(variant) + if (parsed === undefined) return undefined + results.push(parsed) + } + return { + criterion, + holdoutCount, + outcome, + reserveCount, + variants: results, + } +} + +function variantResult(value: unknown): ExperimentVariantResult | undefined { + if (!isRecord(value)) return undefined + const { + assigned, + delivered, + key, + label, + outcomeRate, + read, + replied, + sent, + weight, + } = value + if ( + typeof key !== 'string' || + typeof label !== 'string' || + !isFiniteNumber(outcomeRate) || + !isFiniteNumber(weight) || + ![assigned, delivered, read, replied, sent].every(isCount) + ) { + return undefined + } + return { + assigned: assigned as number, + delivered: delivered as number, + key, + label, + outcomeRate, + read: read as number, + replied: replied as number, + sent: sent as number, + weight, + } +} + +function experimentOutcome(value: unknown): ExperimentOutcome | undefined { + if (!isRecord(value)) return undefined + if (value.state === 'promoted' && typeof value.winnerKey === 'string') { + return {state: 'promoted', winnerKey: value.winnerKey} + } + if (value.state === 'inconclusive' && typeof value.reason === 'string') { + return {reason: value.reason, state: 'inconclusive'} + } + return undefined +} + +function variantSummaries(value: unknown): VariantSummary[] | undefined { + if (!Array.isArray(value)) return undefined + const summaries: VariantSummary[] = [] + for (const variant of value) { + if ( + !isRecord(variant) || + typeof variant.key !== 'string' || + typeof variant.label !== 'string' + ) { + return undefined + } + summaries.push({key: variant.key, label: variant.label}) + } + return summaries +} + +function field(value: object, name: string): unknown { + return (value as Readonly>)[name] +} + +function isCount(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/services/campaigns.ts b/src/services/campaigns.ts index 89a2b155..94a5aae3 100644 --- a/src/services/campaigns.ts +++ b/src/services/campaigns.ts @@ -18,6 +18,11 @@ import { type ResponseMetadata, } from '@polymorfa/sdk' +import { + renderCampaignBetas, + renderExperimentAnalytics, +} from './campaign-experiments.js' + export interface CampaignResult { readonly data: T readonly metadata: { @@ -219,6 +224,7 @@ export function renderCampaign(campaign: Campaign): string { `Failed\t${campaign.failedCount}`, `Skipped\t${campaign.skippedCount}`, `Scheduled\t${formatCampaignTime(campaign.scheduledAt)}`, + ...renderCampaignBetas(campaign), ].join('\n') } @@ -233,6 +239,7 @@ export function renderCampaignAnalytics(analytics: CampaignAnalytics): string { `Skipped\t${analytics.skippedCount}`, `Responded\t${analytics.respondedCount}`, `Response rate\t${analytics.responseRate}`, + ...renderExperimentAnalytics(analytics), ].join('\n') } diff --git a/test/commands/campaign-experiments.test.ts b/test/commands/campaign-experiments.test.ts new file mode 100644 index 00000000..ebb8526c --- /dev/null +++ b/test/commands/campaign-experiments.test.ts @@ -0,0 +1,327 @@ +import {runCommand} from '@oclif/test' +import {afterEach, describe, expect, test} from 'vitest' + +import { + startHttpServer, + withEnvironment, + type TestHttpServer, +} from '../helpers/http-server.js' + +const API_KEY = + 'pmfa_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + +describe('campaign A/B test and weighted variation output (beta)', () => { + let server: TestHttpServer | undefined + + afterEach(async () => { + await server?.close() + server = undefined + }) + + test('renders a promoted A/B test with the winner marked', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 50, + outcome: {state: 'promoted', winnerKey: 'b'}, + reserveCount: 700, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.5625)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toBe( + [ + ...BASE_ANALYTICS_LINES, + '', + 'A/B criterion\tRead rate', + 'A/B outcome\tWinner: Long [b], promoted to the reserve', + 'Holdout\t50', + 'Reserve\t700', + '', + 'Variant\tRead rate\tSent\tAssigned\tDelivered\tRead\tReplied', + 'Short [a]\t40.0%\t80\t100\t75\t32\t6', + 'Long [b] (winner)\t56.3%\t80\t100\t75\t32\t6', + ].join('\n'), + ) + }) + + test('renders an inconclusive A/B test with its reason', async () => { + const data = analytics({ + criterion: 'reply', + holdoutCount: 0, + outcome: {reason: 'insufficient_evidence', state: 'inconclusive'}, + reserveCount: 800, + variants: [variant('a', 'Short', 0.05), variant('b', 'Long', 0.06)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B criterion\tReply rate') + expect(output).toContain( + 'A/B outcome\tInconclusive (insufficient_evidence); the reserve was not sent', + ) + expect(output).toContain( + 'Variant\tReply rate\tSent\tAssigned\tDelivered\tRead\tReplied', + ) + expect(output).not.toContain('(winner)') + }) + + test('renders a running A/B test with no winner yet', async () => { + const data = analytics({ + criterion: 'delivery', + holdoutCount: 10, + outcome: null, + reserveCount: 500, + variants: [variant('a', 'Short', 0.9), variant('b', 'Long', 0.85)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B outcome\tNo winner yet') + expect(output).toContain('Short [a]\t90.0%') + expect(output).not.toContain('(winner)') + }) + + test.each([ + ['null', {experiment: null}], + ['absent', {}], + ])( + 'keeps analytics unchanged when experiment is %s', + async (_name, extra) => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + ...extra, + }) + + expect(output).toBe(BASE_ANALYTICS_LINES.join('\n')) + }, + ) + + test('reports an unreadable experiment instead of printing part of it', async () => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + experiment: {criterion: 'read', variants: 'unexpected'}, + }) + + expect(output).toContain( + 'A/B test\tUnreadable in this CLI version; use --json for the full response.', + ) + }) + + test('escapes control characters in variant labels', async () => { + const output = await humanOutput( + 'campaign analytics campaign_1', + analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: null, + reserveCount: 0, + variants: [variant('a', 'Bad\u001b[31m', 0.1)], + }), + ) + + expect(output).toContain('Bad\\u001b[31m [a]') + expect(output).not.toContain('\u001b') + }) + + test('passes experiment results through unchanged in JSON output', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: {state: 'promoted', winnerKey: 'a'}, + reserveCount: 10, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.2)], + }) + server = await startHttpServer({body: {data, success: true}}) + + const result = await withApiKey(() => + runCommand('campaign analytics campaign_1 --project-slug launch --json'), + ) + + expect(result.error).toBeUndefined() + expect((JSON.parse(result.stdout) as {data: unknown}).data).toEqual(data) + }) + + test('shows the variant count and outcome on an A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: {state: 'promoted', winnerKey: 'a'}, + variants: [campaignVariant('a', 'Short'), campaignVariant('b', 'Long')], + variantStrategy: { + autoPromote: true, + holdoutPercent: 5, + testWindowMinutes: 120, + winnerCriterion: 'read', + }, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'A/B test\t2 variants', + 'A/B outcome\tWinner: Short [a], promoted to the reserve', + ].join('\n'), + ) + }) + + test('shows no winner yet on an undecided A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + variants: [ + campaignVariant('a', 'Short'), + campaignVariant('b', 'Long'), + campaignVariant('c', 'Emoji'), + ], + }) + + expect(output).toContain('A/B test\t3 variants') + expect(output).toContain('A/B outcome\tNo winner yet') + }) + + test('shows the number of weighted alternatives', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + messageVariations: [ + {blueprint: {source: 'Hello!', version: 2}, key: 'hello', weight: 60}, + {blueprint: {source: 'Hi!', version: 2}, key: 'hi', weight: 40}, + ], + variants: null, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'Message variations\t2 weighted alternatives', + ].join('\n'), + ) + }) + + test('keeps a broadcast campaign view unchanged', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + messageVariations: null, + variantStrategy: null, + variants: null, + }) + + expect(output).toBe(BASE_CAMPAIGN_LINES.join('\n')) + }) + + test('mentions both beta input fields in create help', async () => { + const result = await runCommand('campaign create --help') + + expect(result.stdout).toContain('variantStrategy') + expect(result.stdout).toContain('messageVariations') + expect(result.stdout).toContain('403 feature_unavailable') + }) + + async function humanOutput(command: string, data: unknown): Promise { + server = await startHttpServer({body: {data, success: true}}) + const result = await withApiKey(() => + runCommand(`${command} --project-slug launch`), + ) + expect(result.error).toBeUndefined() + return result.stdout.trimEnd() + } + + async function withApiKey(run: () => Promise): Promise { + if (server === undefined) throw new Error('Test server was not started.') + return withEnvironment( + { + POLYMORFA_API_KEY: API_KEY, + POLYMORFA_BASE_URL: server.baseUrl, + POLYMORFA_CLIENT_TOKEN: undefined, + }, + run, + ) + } +}) + +const BASE_ANALYTICS_LINES = [ + 'Campaign\tcampaign_1', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Responded\t12', + 'Response rate\t0.075', +] + +const BASE_CAMPAIGN_LINES = [ + 'ID\tcampaign_1', + 'Name\tOctober offer', + 'Status\trunning', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Scheduled\t2024-08-07T03:07:30.000Z', +] + +function baseAnalytics() { + return { + campaignId: 'campaign_1', + deliveredCount: 150, + failedCount: 0, + readCount: 64, + recipientCount: 1000, + respondedCount: 12, + responseRate: 0.075, + sentCount: 160, + skippedCount: 50, + } +} + +function analytics(experiment: unknown) { + return {...baseAnalytics(), experiment} +} + +function variant(key: string, label: string, outcomeRate: number) { + return { + assigned: 100, + delivered: 75, + key, + label, + outcomeRate, + read: 32, + replied: 6, + sent: 80, + weight: 50, + } +} + +function campaignVariant(key: string, label: string) { + return { + blueprint: {source: `Message ${key}`, version: 2}, + key, + label, + weight: 50, + } +} + +function campaign() { + return { + completedAt: null, + createdAt: 1_723_000_000_000, + deliveredCount: 150, + failedCount: 0, + id: 'campaign_1', + launchedAt: 1_723_000_100_000, + name: 'October offer', + readCount: 64, + recipientCount: 1000, + recipientListId: 'recipients_1', + scheduledAt: 1_723_000_050_000, + sentCount: 160, + skippedCount: 50, + status: 'running', + templateId: null, + updatedAt: 1_723_000_200_000, + } +}