diff --git a/README.md b/README.md index da726c3c..e36af8da 100644 --- a/README.md +++ b/README.md @@ -1714,6 +1714,69 @@ Requeue also accepts the header, but the API does not deduplicate it. Each invocation moves recipients eligible at that time, so a retry can move newly failed recipients rather than returning the first result. +### A/B tests and weighted message variation (beta) + +Both are betas. `campaign create` sends the `--input` object as the request +body, so you set them with these fields. A team that is not enrolled gets +`403 feature_unavailable`. There is no campaign update command; create the +campaign with the fields you need. + +- **A/B test:** `variants` holds two to four messages, each with a `key`, a + `label`, an integer `weight` (the weights add up to 100) and a `blueprint`. + `variantStrategy` sets `winnerCriterion` (`delivery`, `read` or `reply`), + `holdoutPercent` (0 to 30), an optional `testSlicePercent` (5 to 50, + default 20), `autoPromote: true` and `testWindowMinutes` (15 to 1,440). An + A/B test cannot use `templateId`. +- **Weighted message variation:** `messageVariations` holds two to five + alternatives, each with a `key`, an integer `weight` from 1 to 99 (the + weights add up to 100) and a `blueprint`. Each recipient receives one + alternative and keeps it across retries. Every alternative must use the same + `{{variable}}` placeholders. Alternatives send only from linked-device + numbers and cannot be combined with `templateId`, composed messages or an + A/B test. + +```json +{ + "name": "October offer", + "variants": [ + { + "key": "a", + "label": "Short", + "weight": 50, + "blueprint": {"version": 2, "source": "Hi {{first_name | there}}!"} + }, + { + "key": "b", + "label": "Long", + "weight": 50, + "blueprint": { + "version": 2, + "source": "Hello {{first_name | there}}, your offer is ready." + } + } + ], + "variantStrategy": { + "winnerCriterion": "read", + "holdoutPercent": 5, + "autoPromote": true, + "testWindowMinutes": 120 + } +} +``` + +At launch, recipients are split into an optional holdout, a test group that +receives the variants and a reserve that waits for the winner. When the test +window ends, the winner is sent to the reserve. If no variant is clearly ahead, +the outcome is inconclusive and the campaign pauses with the reserve unsent. + +`campaign view` shows the variant count and outcome of an A/B test, and the +number of weighted alternatives. `campaign analytics` adds the criterion, the +outcome (the winner, inconclusive with its reason, or no winner yet), the +holdout and reserve counts, and a row per variant with the criterion rate, +sent, assigned, delivered, read and replied counts. The winner row is marked +`(winner)`. `--json` returns the API's `variants`, `variantStrategy`, +`experimentOutcome`, `messageVariations` and `experiment` fields unchanged. + ### Campaign recipients Recipients are appended to a campaign that has not started sending, and read diff --git a/src/commands/campaign/create.ts b/src/commands/campaign/create.ts index be812749..dfa8806f 100644 --- a/src/commands/campaign/create.ts +++ b/src/commands/campaign/create.ts @@ -6,7 +6,18 @@ import {idempotencyKey} from '../../input/idempotency.js' import {createCampaign, renderCampaign} from '../../services/campaigns.js' export default class CampaignCreate extends BaseCommand { - static override description = 'Create a campaign from one JSON object.' + static override description = `Create a campaign from one JSON object. + +The object is sent as the request body. A/B tests and weighted message variation are betas that need team enrollment; a team that is not enrolled gets 403 feature_unavailable: + +- A/B test: "variants" (two to four, each with key, label, integer weight and blueprint; weights add up to 100) and "variantStrategy" (winnerCriterion delivery, read or reply; holdoutPercent; optional testSlicePercent; autoPromote true; testWindowMinutes). View the results with polymorfa campaign analytics. +- Weighted message variation: "messageVariations", two to five alternatives (key, integer weight, blueprint; weights add up to 100). Each recipient receives one alternative. Linked-device numbers only. + +An A/B test cannot use templateId. messageVariations cannot be combined with templateId, composed messages or an A/B test.` + + static override examples = [ + '<%= config.bin %> <%= command.id %> --project-slug launch-project --input campaign.json', + ] static override flags = { 'idempotency-key': Flags.string({ diff --git a/src/services/campaign-experiments.ts b/src/services/campaign-experiments.ts new file mode 100644 index 00000000..5dc19ae6 --- /dev/null +++ b/src/services/campaign-experiments.ts @@ -0,0 +1,257 @@ +import {terminalText} from '../developer/render.js' + +/* + * Campaign A/B tests and weighted message variation (beta). + * + * The published SDK pin types these fields as unknown or omits them, so the + * CLI reads them as unknown values and narrows them here. The shapes follow + * the SDK's CampaignExperimentResults, CampaignExperimentOutcome, + * CampaignVariant and CampaignMessageVariation types. A value that does not + * match is reported as unreadable instead of being printed partially. + */ + +export type ExperimentOutcome = + | {readonly state: 'promoted'; readonly winnerKey: string} + | {readonly reason: string; readonly state: 'inconclusive'} + +export interface ExperimentVariantResult { + readonly assigned: number + readonly delivered: number + readonly key: string + readonly label: string + readonly outcomeRate: number + readonly read: number + readonly replied: number + readonly sent: number + readonly weight: number +} + +export interface ExperimentResults { + readonly criterion: string + readonly holdoutCount: number + readonly outcome: ExperimentOutcome | null + readonly reserveCount: number + readonly variants: readonly ExperimentVariantResult[] +} + +interface VariantSummary { + readonly key: string + readonly label: string +} + +const UNREADABLE = + 'Unreadable in this CLI version; use --json for the full response.' + +const CRITERION_RATE: Readonly> = { + delivery: 'Delivery rate', + read: 'Read rate', + reply: 'Reply rate', +} + +/** Human lines for `campaign analytics`; empty when the campaign is not an A/B test. */ +export function renderExperimentAnalytics(analytics: object): string[] { + const value = field(analytics, 'experiment') + if (value === undefined || value === null) return [] + const experiment = experimentResults(value) + if (experiment === undefined) return ['', `A/B test\t${UNREADABLE}`] + + const rateLabel = + CRITERION_RATE[experiment.criterion] ?? + `${terminalText(experiment.criterion)} rate` + const winnerKey = + experiment.outcome?.state === 'promoted' + ? experiment.outcome.winnerKey + : undefined + + return [ + '', + `A/B criterion\t${rateLabel}`, + `A/B outcome\t${describeOutcome(experiment.outcome, experiment.variants)}`, + `Holdout\t${experiment.holdoutCount}`, + `Reserve\t${experiment.reserveCount}`, + '', + [ + 'Variant', + rateLabel, + 'Sent', + 'Assigned', + 'Delivered', + 'Read', + 'Replied', + ].join('\t'), + ...experiment.variants.map((variant) => + [ + `${variantName(variant)}${variant.key === winnerKey ? ' (winner)' : ''}`, + percent(variant.outcomeRate), + variant.sent, + variant.assigned, + variant.delivered, + variant.read, + variant.replied, + ].join('\t'), + ), + ] +} + +/** Human lines for `campaign view` covering A/B variants and weighted alternatives. */ +export function renderCampaignBetas(campaign: object): string[] { + const lines: string[] = [] + + const variants = field(campaign, 'variants') + if (variants !== undefined && variants !== null) { + const summaries = variantSummaries(variants) + const outcome = field(campaign, 'experimentOutcome') + const parsed = + outcome === undefined || outcome === null + ? null + : experimentOutcome(outcome) + if (summaries === undefined || parsed === undefined) { + lines.push(`A/B test\t${UNREADABLE}`) + } else { + lines.push( + `A/B test\t${summaries.length} ${summaries.length === 1 ? 'variant' : 'variants'}`, + `A/B outcome\t${describeOutcome(parsed, summaries)}`, + ) + } + } + + const variations = field(campaign, 'messageVariations') + if (variations !== undefined && variations !== null) { + lines.push( + Array.isArray(variations) + ? `Message variations\t${variations.length} weighted ${variations.length === 1 ? 'alternative' : 'alternatives'}` + : `Message variations\t${UNREADABLE}`, + ) + } + + return lines +} + +function describeOutcome( + outcome: ExperimentOutcome | null, + variants: readonly VariantSummary[], +): string { + if (outcome === null) return 'No winner yet' + if (outcome.state === 'promoted') { + const winner = variants.find((variant) => variant.key === outcome.winnerKey) + return `Winner: ${winner === undefined ? terminalText(outcome.winnerKey) : variantName(winner)}, promoted to the reserve` + } + return `Inconclusive (${terminalText(outcome.reason)}); the reserve was not sent` +} + +function variantName(variant: VariantSummary): string { + return `${terminalText(variant.label)} [${terminalText(variant.key)}]` +} + +function percent(rate: number): string { + return `${(Math.round(rate * 1000) / 10).toFixed(1)}%` +} + +function experimentResults(value: unknown): ExperimentResults | undefined { + if (!isRecord(value)) return undefined + const {criterion, holdoutCount, reserveCount, variants} = value + const outcome = + value.outcome === null || value.outcome === undefined + ? null + : experimentOutcome(value.outcome) + if ( + typeof criterion !== 'string' || + !isCount(holdoutCount) || + !isCount(reserveCount) || + outcome === undefined || + !Array.isArray(variants) + ) { + return undefined + } + const results: ExperimentVariantResult[] = [] + for (const variant of variants) { + const parsed = variantResult(variant) + if (parsed === undefined) return undefined + results.push(parsed) + } + return { + criterion, + holdoutCount, + outcome, + reserveCount, + variants: results, + } +} + +function variantResult(value: unknown): ExperimentVariantResult | undefined { + if (!isRecord(value)) return undefined + const { + assigned, + delivered, + key, + label, + outcomeRate, + read, + replied, + sent, + weight, + } = value + if ( + typeof key !== 'string' || + typeof label !== 'string' || + !isFiniteNumber(outcomeRate) || + !isFiniteNumber(weight) || + ![assigned, delivered, read, replied, sent].every(isCount) + ) { + return undefined + } + return { + assigned: assigned as number, + delivered: delivered as number, + key, + label, + outcomeRate, + read: read as number, + replied: replied as number, + sent: sent as number, + weight, + } +} + +function experimentOutcome(value: unknown): ExperimentOutcome | undefined { + if (!isRecord(value)) return undefined + if (value.state === 'promoted' && typeof value.winnerKey === 'string') { + return {state: 'promoted', winnerKey: value.winnerKey} + } + if (value.state === 'inconclusive' && typeof value.reason === 'string') { + return {reason: value.reason, state: 'inconclusive'} + } + return undefined +} + +function variantSummaries(value: unknown): VariantSummary[] | undefined { + if (!Array.isArray(value)) return undefined + const summaries: VariantSummary[] = [] + for (const variant of value) { + if ( + !isRecord(variant) || + typeof variant.key !== 'string' || + typeof variant.label !== 'string' + ) { + return undefined + } + summaries.push({key: variant.key, label: variant.label}) + } + return summaries +} + +function field(value: object, name: string): unknown { + return (value as Readonly>)[name] +} + +function isCount(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/services/campaigns.ts b/src/services/campaigns.ts index 89a2b155..94a5aae3 100644 --- a/src/services/campaigns.ts +++ b/src/services/campaigns.ts @@ -18,6 +18,11 @@ import { type ResponseMetadata, } from '@polymorfa/sdk' +import { + renderCampaignBetas, + renderExperimentAnalytics, +} from './campaign-experiments.js' + export interface CampaignResult { readonly data: T readonly metadata: { @@ -219,6 +224,7 @@ export function renderCampaign(campaign: Campaign): string { `Failed\t${campaign.failedCount}`, `Skipped\t${campaign.skippedCount}`, `Scheduled\t${formatCampaignTime(campaign.scheduledAt)}`, + ...renderCampaignBetas(campaign), ].join('\n') } @@ -233,6 +239,7 @@ export function renderCampaignAnalytics(analytics: CampaignAnalytics): string { `Skipped\t${analytics.skippedCount}`, `Responded\t${analytics.respondedCount}`, `Response rate\t${analytics.responseRate}`, + ...renderExperimentAnalytics(analytics), ].join('\n') } diff --git a/test/commands/campaign-experiments.test.ts b/test/commands/campaign-experiments.test.ts new file mode 100644 index 00000000..ebb8526c --- /dev/null +++ b/test/commands/campaign-experiments.test.ts @@ -0,0 +1,327 @@ +import {runCommand} from '@oclif/test' +import {afterEach, describe, expect, test} from 'vitest' + +import { + startHttpServer, + withEnvironment, + type TestHttpServer, +} from '../helpers/http-server.js' + +const API_KEY = + 'pmfa_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + +describe('campaign A/B test and weighted variation output (beta)', () => { + let server: TestHttpServer | undefined + + afterEach(async () => { + await server?.close() + server = undefined + }) + + test('renders a promoted A/B test with the winner marked', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 50, + outcome: {state: 'promoted', winnerKey: 'b'}, + reserveCount: 700, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.5625)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toBe( + [ + ...BASE_ANALYTICS_LINES, + '', + 'A/B criterion\tRead rate', + 'A/B outcome\tWinner: Long [b], promoted to the reserve', + 'Holdout\t50', + 'Reserve\t700', + '', + 'Variant\tRead rate\tSent\tAssigned\tDelivered\tRead\tReplied', + 'Short [a]\t40.0%\t80\t100\t75\t32\t6', + 'Long [b] (winner)\t56.3%\t80\t100\t75\t32\t6', + ].join('\n'), + ) + }) + + test('renders an inconclusive A/B test with its reason', async () => { + const data = analytics({ + criterion: 'reply', + holdoutCount: 0, + outcome: {reason: 'insufficient_evidence', state: 'inconclusive'}, + reserveCount: 800, + variants: [variant('a', 'Short', 0.05), variant('b', 'Long', 0.06)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B criterion\tReply rate') + expect(output).toContain( + 'A/B outcome\tInconclusive (insufficient_evidence); the reserve was not sent', + ) + expect(output).toContain( + 'Variant\tReply rate\tSent\tAssigned\tDelivered\tRead\tReplied', + ) + expect(output).not.toContain('(winner)') + }) + + test('renders a running A/B test with no winner yet', async () => { + const data = analytics({ + criterion: 'delivery', + holdoutCount: 10, + outcome: null, + reserveCount: 500, + variants: [variant('a', 'Short', 0.9), variant('b', 'Long', 0.85)], + }) + + const output = await humanOutput('campaign analytics campaign_1', data) + + expect(output).toContain('A/B outcome\tNo winner yet') + expect(output).toContain('Short [a]\t90.0%') + expect(output).not.toContain('(winner)') + }) + + test.each([ + ['null', {experiment: null}], + ['absent', {}], + ])( + 'keeps analytics unchanged when experiment is %s', + async (_name, extra) => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + ...extra, + }) + + expect(output).toBe(BASE_ANALYTICS_LINES.join('\n')) + }, + ) + + test('reports an unreadable experiment instead of printing part of it', async () => { + const output = await humanOutput('campaign analytics campaign_1', { + ...baseAnalytics(), + experiment: {criterion: 'read', variants: 'unexpected'}, + }) + + expect(output).toContain( + 'A/B test\tUnreadable in this CLI version; use --json for the full response.', + ) + }) + + test('escapes control characters in variant labels', async () => { + const output = await humanOutput( + 'campaign analytics campaign_1', + analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: null, + reserveCount: 0, + variants: [variant('a', 'Bad\u001b[31m', 0.1)], + }), + ) + + expect(output).toContain('Bad\\u001b[31m [a]') + expect(output).not.toContain('\u001b') + }) + + test('passes experiment results through unchanged in JSON output', async () => { + const data = analytics({ + criterion: 'read', + holdoutCount: 0, + outcome: {state: 'promoted', winnerKey: 'a'}, + reserveCount: 10, + variants: [variant('a', 'Short', 0.4), variant('b', 'Long', 0.2)], + }) + server = await startHttpServer({body: {data, success: true}}) + + const result = await withApiKey(() => + runCommand('campaign analytics campaign_1 --project-slug launch --json'), + ) + + expect(result.error).toBeUndefined() + expect((JSON.parse(result.stdout) as {data: unknown}).data).toEqual(data) + }) + + test('shows the variant count and outcome on an A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: {state: 'promoted', winnerKey: 'a'}, + variants: [campaignVariant('a', 'Short'), campaignVariant('b', 'Long')], + variantStrategy: { + autoPromote: true, + holdoutPercent: 5, + testWindowMinutes: 120, + winnerCriterion: 'read', + }, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'A/B test\t2 variants', + 'A/B outcome\tWinner: Short [a], promoted to the reserve', + ].join('\n'), + ) + }) + + test('shows no winner yet on an undecided A/B campaign', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + variants: [ + campaignVariant('a', 'Short'), + campaignVariant('b', 'Long'), + campaignVariant('c', 'Emoji'), + ], + }) + + expect(output).toContain('A/B test\t3 variants') + expect(output).toContain('A/B outcome\tNo winner yet') + }) + + test('shows the number of weighted alternatives', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + messageVariations: [ + {blueprint: {source: 'Hello!', version: 2}, key: 'hello', weight: 60}, + {blueprint: {source: 'Hi!', version: 2}, key: 'hi', weight: 40}, + ], + variants: null, + }) + + expect(output).toBe( + [ + ...BASE_CAMPAIGN_LINES, + 'Message variations\t2 weighted alternatives', + ].join('\n'), + ) + }) + + test('keeps a broadcast campaign view unchanged', async () => { + const output = await humanOutput('campaign view campaign_1', { + ...campaign(), + experimentOutcome: null, + messageVariations: null, + variantStrategy: null, + variants: null, + }) + + expect(output).toBe(BASE_CAMPAIGN_LINES.join('\n')) + }) + + test('mentions both beta input fields in create help', async () => { + const result = await runCommand('campaign create --help') + + expect(result.stdout).toContain('variantStrategy') + expect(result.stdout).toContain('messageVariations') + expect(result.stdout).toContain('403 feature_unavailable') + }) + + async function humanOutput(command: string, data: unknown): Promise { + server = await startHttpServer({body: {data, success: true}}) + const result = await withApiKey(() => + runCommand(`${command} --project-slug launch`), + ) + expect(result.error).toBeUndefined() + return result.stdout.trimEnd() + } + + async function withApiKey(run: () => Promise): Promise { + if (server === undefined) throw new Error('Test server was not started.') + return withEnvironment( + { + POLYMORFA_API_KEY: API_KEY, + POLYMORFA_BASE_URL: server.baseUrl, + POLYMORFA_CLIENT_TOKEN: undefined, + }, + run, + ) + } +}) + +const BASE_ANALYTICS_LINES = [ + 'Campaign\tcampaign_1', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Responded\t12', + 'Response rate\t0.075', +] + +const BASE_CAMPAIGN_LINES = [ + 'ID\tcampaign_1', + 'Name\tOctober offer', + 'Status\trunning', + 'Recipients\t1000', + 'Sent\t160', + 'Delivered\t150', + 'Read\t64', + 'Failed\t0', + 'Skipped\t50', + 'Scheduled\t2024-08-07T03:07:30.000Z', +] + +function baseAnalytics() { + return { + campaignId: 'campaign_1', + deliveredCount: 150, + failedCount: 0, + readCount: 64, + recipientCount: 1000, + respondedCount: 12, + responseRate: 0.075, + sentCount: 160, + skippedCount: 50, + } +} + +function analytics(experiment: unknown) { + return {...baseAnalytics(), experiment} +} + +function variant(key: string, label: string, outcomeRate: number) { + return { + assigned: 100, + delivered: 75, + key, + label, + outcomeRate, + read: 32, + replied: 6, + sent: 80, + weight: 50, + } +} + +function campaignVariant(key: string, label: string) { + return { + blueprint: {source: `Message ${key}`, version: 2}, + key, + label, + weight: 50, + } +} + +function campaign() { + return { + completedAt: null, + createdAt: 1_723_000_000_000, + deliveredCount: 150, + failedCount: 0, + id: 'campaign_1', + launchedAt: 1_723_000_100_000, + name: 'October offer', + readCount: 64, + recipientCount: 1000, + recipientListId: 'recipients_1', + scheduledAt: 1_723_000_050_000, + sentCount: 160, + skippedCount: 50, + status: 'running', + templateId: null, + updatedAt: 1_723_000_200_000, + } +}