Show campaign A/B results and weighted variations in CLI output

README.md

@@ -1714,6 +1714,69 @@ Requeue also accepts the header, but the API does not deduplicate it. Each
 invocation moves recipients eligible at that time, so a retry can move newly
 failed recipients rather than returning the first result.
 
+### A/B tests and weighted message variation (beta)
+
+Both are betas. `campaign create` sends the `--input` object as the request
+body, so you set them with these fields. A team that is not enrolled gets
+`403 feature_unavailable`. There is no campaign update command; create the
+campaign with the fields you need.
+
+- **A/B test:** `variants` holds two to four messages, each with a `key`, a
+  `label`, an integer `weight` (the weights add up to 100) and a `blueprint`.
+  `variantStrategy` sets `winnerCriterion` (`delivery`, `read` or `reply`),
+  `holdoutPercent` (0 to 30), an optional `testSlicePercent` (5 to 50,
+  default 20), `autoPromote: true` and `testWindowMinutes` (15 to 1,440). An
+  A/B test cannot use `templateId`.
+- **Weighted message variation:** `messageVariations` holds two to five
+  alternatives, each with a `key`, an integer `weight` from 1 to 99 (the
+  weights add up to 100) and a `blueprint`. Each recipient receives one
+  alternative and keeps it across retries. Every alternative must use the same
+  `{{variable}}` placeholders. Alternatives send only from linked-device
+  numbers and cannot be combined with `templateId`, composed messages or an
+  A/B test.
+
+```json
+{
+  "name": "October offer",
+  "variants": [
+    {
+      "key": "a",
+      "label": "Short",
+      "weight": 50,
+      "blueprint": {"version": 2, "source": "Hi {{first_name | there}}!"}
+    },
+    {
+      "key": "b",
+      "label": "Long",
+      "weight": 50,
+      "blueprint": {
+        "version": 2,
+        "source": "Hello {{first_name | there}}, your offer is ready."
+      }
+    }
+  ],
+  "variantStrategy": {
+    "winnerCriterion": "read",
+    "holdoutPercent": 5,
+    "autoPromote": true,
+    "testWindowMinutes": 120
+  }
+}
+```
+
+At launch, recipients are split into an optional holdout, a test group that
+receives the variants and a reserve that waits for the winner. When the test
+window ends, the winner is sent to the reserve. If no variant is clearly ahead,
+the outcome is inconclusive and the campaign pauses with the reserve unsent.
+
+`campaign view` shows the variant count and outcome of an A/B test, and the
+number of weighted alternatives. `campaign analytics` adds the criterion, the
+outcome (the winner, inconclusive with its reason, or no winner yet), the
+holdout and reserve counts, and a row per variant with the criterion rate,
+sent, assigned, delivered, read and replied counts. The winner row is marked
+`(winner)`. `--json` returns the API's `variants`, `variantStrategy`,
+`experimentOutcome`, `messageVariations` and `experiment` fields unchanged.
+
 ### Campaign recipients
 
 Recipients are appended to a campaign that has not started sending, and read

src/commands/campaign/create.ts

@@ -6,7 +6,18 @@ import {idempotencyKey} from '../../input/idempotency.js'
 import {createCampaign, renderCampaign} from '../../services/campaigns.js'
 
 export default class CampaignCreate extends BaseCommand {
-  static override description = 'Create a campaign from one JSON object.'
+  static override description = `Create a campaign from one JSON object.
+
+The object is sent as the request body. A/B tests and weighted message variation are betas that need team enrollment; a team that is not enrolled gets 403 feature_unavailable:
+
+- A/B test: "variants" (two to four, each with key, label, integer weight and blueprint; weights add up to 100) and "variantStrategy" (winnerCriterion delivery, read or reply; holdoutPercent; optional testSlicePercent; autoPromote true; testWindowMinutes). View the results with polymorfa campaign analytics.
+- Weighted message variation: "messageVariations", two to five alternatives (key, integer weight, blueprint; weights add up to 100). Each recipient receives one alternative. Linked-device numbers only.
+
+An A/B test cannot use templateId. messageVariations cannot be combined with templateId, composed messages or an A/B test.`
+
+  static override examples = [
+    '<%= config.bin %> <%= command.id %> --project-slug launch-project --input campaign.json',
+  ]
 
   static override flags = {
     'idempotency-key': Flags.string({

src/services/campaign-experiments.ts

@@ -0,0 +1,257 @@
+import {terminalText} from '../developer/render.js'
+
+/*
+ * Campaign A/B tests and weighted message variation (beta).
+ *
+ * The published SDK pin types these fields as unknown or omits them, so the
+ * CLI reads them as unknown values and narrows them here. The shapes follow
+ * the SDK's CampaignExperimentResults, CampaignExperimentOutcome,
+ * CampaignVariant and CampaignMessageVariation types. A value that does not
+ * match is reported as unreadable instead of being printed partially.
+ */
+
+export type ExperimentOutcome =
+  | {readonly state: 'promoted'; readonly winnerKey: string}
+  | {readonly reason: string; readonly state: 'inconclusive'}
+
+export interface ExperimentVariantResult {
+  readonly assigned: number
+  readonly delivered: number
+  readonly key: string
+  readonly label: string
+  readonly outcomeRate: number
+  readonly read: number
+  readonly replied: number
+  readonly sent: number
+  readonly weight: number
+}
+
+export interface ExperimentResults {
+  readonly criterion: string
+  readonly holdoutCount: number
+  readonly outcome: ExperimentOutcome | null
+  readonly reserveCount: number
+  readonly variants: readonly ExperimentVariantResult[]
+}
+
+interface VariantSummary {
+  readonly key: string
+  readonly label: string
+}
+
+const UNREADABLE =
+  'Unreadable in this CLI version; use --json for the full response.'
+
+const CRITERION_RATE: Readonly<Record<string, string>> = {
+  delivery: 'Delivery rate',
+  read: 'Read rate',
+  reply: 'Reply rate',
+}
+
+/** Human lines for `campaign analytics`; empty when the campaign is not an A/B test. */
+export function renderExperimentAnalytics(analytics: object): string[] {
+  const value = field(analytics, 'experiment')
+  if (value === undefined || value === null) return []
+  const experiment = experimentResults(value)
+  if (experiment === undefined) return ['', `A/B test\t${UNREADABLE}`]
+
+  const rateLabel =
+    CRITERION_RATE[experiment.criterion] ??
+    `${terminalText(experiment.criterion)} rate`
+  const winnerKey =
+    experiment.outcome?.state === 'promoted'
+      ? experiment.outcome.winnerKey
+      : undefined
+
+  return [
+    '',
+    `A/B criterion\t${rateLabel}`,
+    `A/B outcome\t${describeOutcome(experiment.outcome, experiment.variants)}`,
+    `Holdout\t${experiment.holdoutCount}`,
+    `Reserve\t${experiment.reserveCount}`,
+    '',
+    [
+      'Variant',
+      rateLabel,
+      'Sent',
+      'Assigned',
+      'Delivered',
+      'Read',
+      'Replied',
+    ].join('\t'),
+    ...experiment.variants.map((variant) =>
+      [
+        `${variantName(variant)}${variant.key === winnerKey ? ' (winner)' : ''}`,
+        percent(variant.outcomeRate),
+        variant.sent,
+        variant.assigned,
+        variant.delivered,
+        variant.read,
+        variant.replied,
+      ].join('\t'),
+    ),
+  ]
+}
+
+/** Human lines for `campaign view` covering A/B variants and weighted alternatives. */
+export function renderCampaignBetas(campaign: object): string[] {
+  const lines: string[] = []
+
+  const variants = field(campaign, 'variants')
+  if (variants !== undefined && variants !== null) {
+    const summaries = variantSummaries(variants)
+    const outcome = field(campaign, 'experimentOutcome')
+    const parsed =
+      outcome === undefined || outcome === null
+        ? null
+        : experimentOutcome(outcome)
+    if (summaries === undefined || parsed === undefined) {
+      lines.push(`A/B test\t${UNREADABLE}`)
+    } else {
+      lines.push(
+        `A/B test\t${summaries.length} ${summaries.length === 1 ? 'variant' : 'variants'}`,
+        `A/B outcome\t${describeOutcome(parsed, summaries)}`,
+      )
+    }
+  }
+
+  const variations = field(campaign, 'messageVariations')
+  if (variations !== undefined && variations !== null) {
+    lines.push(
+      Array.isArray(variations)
+        ? `Message variations\t${variations.length} weighted ${variations.length === 1 ? 'alternative' : 'alternatives'}`
+        : `Message variations\t${UNREADABLE}`,
+    )
+  }
+
+  return lines
+}
+
+function describeOutcome(
+  outcome: ExperimentOutcome | null,
+  variants: readonly VariantSummary[],
+): string {
+  if (outcome === null) return 'No winner yet'
+  if (outcome.state === 'promoted') {
+    const winner = variants.find((variant) => variant.key === outcome.winnerKey)
+    return `Winner: ${winner === undefined ? terminalText(outcome.winnerKey) : variantName(winner)}, promoted to the reserve`
+  }
+  return `Inconclusive (${terminalText(outcome.reason)}); the reserve was not sent`
+}
+
+function variantName(variant: VariantSummary): string {
+  return `${terminalText(variant.label)} [${terminalText(variant.key)}]`
+}
+
+function percent(rate: number): string {
+  return `${(Math.round(rate * 1000) / 10).toFixed(1)}%`
+}
+
+function experimentResults(value: unknown): ExperimentResults | undefined {
+  if (!isRecord(value)) return undefined
+  const {criterion, holdoutCount, reserveCount, variants} = value
+  const outcome =
+    value.outcome === null || value.outcome === undefined
+      ? null
+      : experimentOutcome(value.outcome)
+  if (
+    typeof criterion !== 'string' ||
+    !isCount(holdoutCount) ||
+    !isCount(reserveCount) ||
+    outcome === undefined ||
+    !Array.isArray(variants)
+  ) {
+    return undefined
+  }
+  const results: ExperimentVariantResult[] = []
+  for (const variant of variants) {
+    const parsed = variantResult(variant)
+    if (parsed === undefined) return undefined
+    results.push(parsed)
+  }
+  return {
+    criterion,
+    holdoutCount,
+    outcome,
+    reserveCount,
+    variants: results,
+  }
+}
+
+function variantResult(value: unknown): ExperimentVariantResult | undefined {
+  if (!isRecord(value)) return undefined
+  const {
+    assigned,
+    delivered,
+    key,
+    label,
+    outcomeRate,
+    read,
+    replied,
+    sent,
+    weight,
+  } = value
+  if (
+    typeof key !== 'string' ||
+    typeof label !== 'string' ||
+    !isFiniteNumber(outcomeRate) ||
+    !isFiniteNumber(weight) ||
+    ![assigned, delivered, read, replied, sent].every(isCount)
+  ) {
+    return undefined
+  }
+  return {
+    assigned: assigned as number,
+    delivered: delivered as number,
+    key,
+    label,
+    outcomeRate,
+    read: read as number,
+    replied: replied as number,
+    sent: sent as number,
+    weight,
+  }
+}
+
+function experimentOutcome(value: unknown): ExperimentOutcome | undefined {
+  if (!isRecord(value)) return undefined
+  if (value.state === 'promoted' && typeof value.winnerKey === 'string') {
+    return {state: 'promoted', winnerKey: value.winnerKey}
+  }
+  if (value.state === 'inconclusive' && typeof value.reason === 'string') {
+    return {reason: value.reason, state: 'inconclusive'}
+  }
+  return undefined
+}
+
+function variantSummaries(value: unknown): VariantSummary[] | undefined {
+  if (!Array.isArray(value)) return undefined
+  const summaries: VariantSummary[] = []
+  for (const variant of value) {
+    if (
+      !isRecord(variant) ||
+      typeof variant.key !== 'string' ||
+      typeof variant.label !== 'string'
+    ) {
+      return undefined
+    }
+    summaries.push({key: variant.key, label: variant.label})
+  }
+  return summaries
+}
+
+function field(value: object, name: string): unknown {
+  return (value as Readonly<Record<string, unknown>>)[name]
+}
+
+function isCount(value: unknown): value is number {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
+function isFiniteNumber(value: unknown): value is number {
+  return typeof value === 'number' && Number.isFinite(value)
+}
+
+function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}

src/services/campaigns.ts

@@ -18,6 +18,11 @@ import {
   type ResponseMetadata,
 } from '@polymorfa/sdk'
 
+import {
+  renderCampaignBetas,
+  renderExperimentAnalytics,
+} from './campaign-experiments.js'
+
 export interface CampaignResult<T> {
   readonly data: T
   readonly metadata: {
@@ -219,6 +224,7 @@ export function renderCampaign(campaign: Campaign): string {
     `Failed\t${campaign.failedCount}`,
     `Skipped\t${campaign.skippedCount}`,
     `Scheduled\t${formatCampaignTime(campaign.scheduledAt)}`,
+    ...renderCampaignBetas(campaign),
   ].join('\n')
 }
 
@@ -233,6 +239,7 @@ export function renderCampaignAnalytics(analytics: CampaignAnalytics): string {
     `Skipped\t${analytics.skippedCount}`,
     `Responded\t${analytics.respondedCount}`,
     `Response rate\t${analytics.responseRate}`,
+    ...renderExperimentAnalytics(analytics),
   ].join('\n')
 }