From 67846829e9c039b518e0f68e4aa9bd7c6a8f6e05 Mon Sep 17 00:00:00 2001 From: Aman Varshney Date: Thu, 15 Oct 2026 14:18:53 +0530 Subject: [PATCH 1/3] feat(telemetry): report to the Prisma CLI telemetry backend instead of PostHog create-prisma now sends one event per run to the same backend as the Prisma CLI, in its shape (`command: "create-prisma"`, flag names, database target, exit code, coding agent), with the outcome details in `properties`. It shares the Prisma CLI's consent and installation id, prints its one-time notice, honours `prisma telemetry disable`, PRISMA_DISABLE_TELEMETRY and DO_NOT_TRACK, keeps the CREATE_PRISMA_* opt-outs, and never reports from CI (ci-info). A failed install inside `prisma orm init` now reports the package manager's error code from the CLI's `meta.install.stderrTail`, so the Windows CLI.INIT_INSTALL_FAILED runs can be diagnosed. The PostHog key and host are gone from the build and publish workflow. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish.yml | 6 - README.md | 2 +- bun.lock | 6 + package.json | 2 + scripts/smoke-node.mjs | 2 +- src/create-outcome.ts | 2 + src/tasks/prisma-cli.ts | 19 +++ src/telemetry/client.ts | 221 ++++++++++++++++++---------- src/telemetry/create.ts | 149 ++++++++++--------- tests/telemetry-client.test.ts | 254 ++++++++++++++++++++------------- tests/telemetry.test.ts | 145 ++++++++++++------- tsdown.config.ts | 4 - 12 files changed, 507 insertions(+), 305 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9da31ff..8e8e0ca 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -178,9 +178,6 @@ jobs: - name: Build run: bun run build - env: - CREATE_PRISMA_TELEMETRY_API_KEY: ${{ steps.preview.outputs.publish_next == 'true' && secrets.CREATE_PRISMA_TELEMETRY_API_KEY || '' }} - CREATE_PRISMA_TELEMETRY_HOST: ${{ steps.preview.outputs.publish_next == 'true' && vars.CREATE_PRISMA_TELEMETRY_HOST || '' }} - name: Test packed CLI on the minimum Node version and its bundled npm run: node scripts/smoke-node.mjs @@ -276,9 +273,6 @@ jobs: if: github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):') runs-on: ubuntu-latest timeout-minutes: 15 - env: - CREATE_PRISMA_TELEMETRY_API_KEY: ${{ secrets.CREATE_PRISMA_TELEMETRY_API_KEY }} - CREATE_PRISMA_TELEMETRY_HOST: ${{ vars.CREATE_PRISMA_TELEMETRY_HOST }} permissions: contents: write id-token: write diff --git a/README.md b/README.md index 7067e1d..e3c3a05 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,4 @@ bun run build ## Telemetry -Published builds may send anonymous usage telemetry. It never includes project names, file paths, or database URLs. Failure events carry only stable identifiers such as exit codes and tool error codes, never messages or command output. A run rejected for a non-empty directory reports only whether that directory held an existing app, a Prisma project, or other files. Disable it with `DO_NOT_TRACK`, `CREATE_PRISMA_DISABLE_TELEMETRY`, or `CREATE_PRISMA_TELEMETRY_DISABLED`. +Published builds send anonymous usage telemetry to the same service as the Prisma CLI, and share its consent: the first run that reports prints a one-time notice, and `prisma telemetry disable` turns it off for both. Each run reports one event when it ends, under the installation id the Prisma CLI keeps in its user config, with the outcome, the chosen template, database, authoring style, package manager, the names of the flags you passed and how long the run took. It never includes project names, file paths, flag values, or database URLs. Failure events carry only stable identifiers such as exit codes and tool error codes, never messages or command output. A run rejected for a non-empty directory reports only whether that directory held an existing app, a Prisma project, or other files. Nothing is sent in CI. Disable it with `prisma telemetry disable`, `DO_NOT_TRACK=1`, or `PRISMA_DISABLE_TELEMETRY=1`; `CREATE_PRISMA_DISABLE_TELEMETRY` and `CREATE_PRISMA_TELEMETRY_DISABLED` still work. diff --git a/bun.lock b/bun.lock index 4d26afc..3347a52 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,8 @@ "dependencies": { "@clack/prompts": "^1.8.1", "@effect/platform-node-shared": "4.0.1", + "@vercel/detect-agent": "^1.2.7", + "ci-info": "^4.4.0", "effect": "4.0.1", "execa": "^10.0.1", "handlebars": "^4.7.10", @@ -215,6 +217,8 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "@vercel/detect-agent": ["@vercel/detect-agent@1.2.7", "", {}, "sha512-5tWaC9V+u4SHcSMKBz9FQb3Dhl/1A2lQ6QqHovu8uQeb8dTyjEevpJMANm/uQEn/usYB63fpdjWYjkJ8/bGtjA=="], + "@yuku-codegen/binding-android-arm64": ["@yuku-codegen/binding-android-arm64@0.10.2", "", { "os": "android", "cpu": "arm64" }, "sha512-Oc2KInVkPfUjEB4PeV6X0NvIoyYTzDe/WmqFrTwBqKs6YKnz4A7XwWQvJTb7M0rLz9a6WgpUUiiS5QKaCi4O/g=="], "@yuku-codegen/binding-darwin-arm64": ["@yuku-codegen/binding-darwin-arm64@0.10.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-3H7eNPIHJndUJXZ4QUGBPq1zC6RGcUZfm7bymJes+/N7wTVWUn1bxZ9JcozYHpkWQNm9f23G8YWbaHD9aYH72A=="], @@ -317,6 +321,8 @@ "chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="], + "ci-info": ["ci-info@4.4.0", "", {}, "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg=="], + "citty": ["citty@0.1.6", "", { "dependencies": { "consola": "^3.2.3" } }, "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ=="], "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], diff --git a/package.json b/package.json index f835599..bec81da 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,8 @@ "dependencies": { "@clack/prompts": "^1.8.1", "@effect/platform-node-shared": "4.0.1", + "@vercel/detect-agent": "^1.2.7", + "ci-info": "^4.4.0", "effect": "4.0.1", "execa": "^10.0.1", "handlebars": "^4.7.10" diff --git a/scripts/smoke-node.mjs b/scripts/smoke-node.mjs index 80822d2..0f4eecb 100644 --- a/scripts/smoke-node.mjs +++ b/scripts/smoke-node.mjs @@ -7,7 +7,7 @@ const directory = await mkdtemp(path.join(tmpdir(), "create-prisma-node-smoke-") const env = { DO_NOT_TRACK: "1", CREATE_PRISMA_DISABLE_TELEMETRY: "1", - PRISMA_NEXT_DISABLE_TELEMETRY: "1", + PRISMA_DISABLE_TELEMETRY: "1", NO_UPDATE_NOTIFIER: "1", npm_config_engine_strict: "false", }; diff --git a/src/create-outcome.ts b/src/create-outcome.ts index 442125e..350d564 100644 --- a/src/create-outcome.ts +++ b/src/create-outcome.ts @@ -116,6 +116,8 @@ export class PrismaCliCommandError extends Schema.TaggedError diagnostic.severity === "error"); const summary = failure?.summary ?? failure?.message; + const packageManagerErrorCode = getInstallErrorCode(failure); return yield* new PrismaCliCommandError({ message: redactSecrets( [summary, failure?.why].filter(Boolean).join(": ") || @@ -95,6 +113,7 @@ export const runPrismaJsonCommandEffect = Effect.fn("PrismaCli.runJson")(functio ? { command: envelope.commandId ?? envelope.command } : {}), ...(failure?.code ? { code: failure.code } : {}), + ...(packageManagerErrorCode ? { packageManagerErrorCode } : {}), stderr: redactSecrets(result.stderr), exitCode: result.exitCode, childProcessFailure: result.childProcessFailure, diff --git a/src/telemetry/client.ts b/src/telemetry/client.ts index 939e663..18e6e2a 100644 --- a/src/telemetry/client.ts +++ b/src/telemetry/client.ts @@ -1,123 +1,190 @@ -import { Effect, FileSystem, Schema } from "effect"; +import { determineAgent } from "@vercel/detect-agent"; +import ci from "ci-info"; +import { Effect, FileSystem } from "effect"; import { randomUUID } from "node:crypto"; -import os from "node:os"; import path from "node:path"; -import { applicationRuntime } from "../runtime"; - -const TELEMETRY_API_KEY = process.env.CREATE_PRISMA_TELEMETRY_API_KEY ?? ""; -const TELEMETRY_HOST = process.env.CREATE_PRISMA_TELEMETRY_HOST || "https://us.i.posthog.com"; -const TELEMETRY_CONFIG_FILE = "telemetry.json"; +// The Prisma CLI's anonymous usage backend (`prisma/orm` apps/telemetry-backend). +// create-prisma rows share its table and are told apart by their `command`. +const TELEMETRY_BACKEND_URL = "https://cmpbfbsdp09hr3jf7pojjs5qs.ewr.prisma.build"; +const TELEMETRY_DOCS_URL = "https://www.prisma.io/docs/cli"; export const TELEMETRY_TIMEOUT_MS = 2_000; -const AnonymousId = Schema.String.check(Schema.isUUID(4)); -const decodeAnonymousId = Schema.decodeUnknownExit(AnonymousId); type TelemetryValue = boolean | number | string | string[] | null | undefined; -type TelemetryProperties = Record; +export type TelemetryProperties = Record; -const isTruthyEnvValue = (value: string | undefined) => - ["1", "true", "yes", "on"].includes( - String(value ?? "") - .trim() - .toLowerCase(), - ); +export type TelemetryEvent = { + command: string; + flags: string[]; + databaseTarget: string | null; + exitCode: number; + properties: TelemetryProperties; +}; + +type PrismaUserConfig = Record & { + enableTelemetry?: unknown; + installationId?: unknown; +}; + +const getCliVersion = () => process.env.CREATE_PRISMA_CLI_VERSION ?? "0.0.0"; -function shouldDisableTelemetry(): boolean { +// The Prisma CLI's spelling: set-but-falsy values ('', '0', 'false') do not opt out. +const isTruthyOptOut = (value: string | undefined) => + value !== undefined && !["", "0", "false"].includes(value.trim().toLowerCase()); + +function isTelemetryDisabledByEnvironment(env: NodeJS.ProcessEnv = process.env): boolean { return ( - TELEMETRY_API_KEY.length === 0 || - isTruthyEnvValue(process.env.CI) || - isTruthyEnvValue(process.env.GITHUB_ACTIONS) || - process.env.CREATE_PRISMA_DISABLE_TELEMETRY !== undefined || - process.env.CREATE_PRISMA_TELEMETRY_DISABLED !== undefined || - process.env.DO_NOT_TRACK !== undefined + // Unbuilt source runs (tests, `bun run dev`) carry no release to attribute. + getCliVersion() === "0.0.0" || + ci.isCI || + isTruthyOptOut(env.PRISMA_DISABLE_TELEMETRY) || + isTruthyOptOut(env.DO_NOT_TRACK) || + // create-prisma's own opt-outs from before it shared the Prisma CLI's consent. + env.CREATE_PRISMA_DISABLE_TELEMETRY !== undefined || + env.CREATE_PRISMA_TELEMETRY_DISABLED !== undefined ); } -function getTelemetryConfigDir(): string { - if (process.platform === "darwin") { - return path.join(os.homedir(), "Library", "Application Support", "create-prisma"); - } +// The Prisma CLI's preference store, so `prisma telemetry disable` covers create-prisma +// and both report under one installation id. +function getPrismaUserConfigPath(env: NodeJS.ProcessEnv = process.env): string | undefined { + const set = (value: string | undefined) => (value ? value : undefined); if (process.platform === "win32") { - return path.join( - process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"), - "create-prisma", - ); + const appData = set(env.APPDATA); + if (appData) return path.join(appData, "prisma", "config.json"); + const userProfile = set(env.USERPROFILE); + return userProfile + ? path.join(userProfile, "AppData", "Roaming", "prisma", "config.json") + : undefined; } - return path.join( - process.env.XDG_CONFIG_HOME ?? path.join(os.homedir(), ".config"), - "create-prisma", - ); + const xdg = set(env.XDG_CONFIG_HOME); + if (xdg) return path.join(xdg, "prisma", "config.json"); + const home = set(env.HOME); + return home ? path.join(home, ".config", "prisma", "config.json") : undefined; } -const getAnonymousIdEffect = Effect.fn("Telemetry.getAnonymousId")(function* () { +const firstRunNotice = [ + "Prisma collects anonymous CLI usage data, enabled by default.", + `What's collected and why: ${TELEMETRY_DOCS_URL}.`, + 'Opt out: run "prisma telemetry disable", set DO_NOT_TRACK=1 or', + "PRISMA_DISABLE_TELEMETRY=1.", +].join(" "); + +const readPrismaUserConfigEffect = Effect.fn("Telemetry.readUserConfig")(function* ( + configPath: string, +) { const fs = yield* FileSystem.FileSystem; - const configPath = path.join(getTelemetryConfigDir(), TELEMETRY_CONFIG_FILE); - const persisted = yield* fs.readFileString(configPath).pipe( + return yield* fs.readFileString(configPath).pipe( Effect.flatMap((source) => Effect.try({ - try: () => JSON.parse(source) as { anonymousId?: unknown }, - catch: () => ({ anonymousId: undefined }), + try: () => JSON.parse(source) as unknown, + catch: () => undefined, }), ), - Effect.catch(() => Effect.succeed({ anonymousId: undefined })), + Effect.map((parsed): PrismaUserConfig => + parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as PrismaUserConfig) + : {}, + ), + Effect.catch(() => Effect.succeed({})), ); - const decoded = decodeAnonymousId(persisted.anonymousId); - if (decoded._tag === "Success") return decoded.value; +}); - const anonymousId = randomUUID(); - yield* Effect.gen(function* () { +// Discloses once, then stores the id beside the existing preferences (temp file + rename, +// as the Prisma CLI writes it). A failed write sends nothing and discloses again next run. +const discloseAndMintEffect = Effect.fn("Telemetry.discloseAndMint")(function* ( + configPath: string, + config: PrismaUserConfig, +) { + const fs = yield* FileSystem.FileSystem; + yield* Effect.sync(() => process.stderr.write(`${firstRunNotice}\n`)); + const installationId = randomUUID(); + const tmpPath = `${configPath}.${process.pid}.tmp`; + return yield* Effect.gen(function* () { yield* fs.makeDirectory(path.dirname(configPath), { recursive: true }); - yield* fs.writeFileString(configPath, `${JSON.stringify({ anonymousId }, null, 2)}\n`); - }).pipe(Effect.catch(() => Effect.void)); - return anonymousId; + yield* fs.writeFileString( + tmpPath, + `${JSON.stringify({ ...config, installationId }, null, 2)}\n`, + ); + yield* fs.rename(tmpPath, configPath); + return installationId; + }).pipe(Effect.catch(() => Effect.succeed(undefined))); }); -const getCommonProperties = (): TelemetryProperties => ({ - "cli-version": process.env.CREATE_PRISMA_CLI_VERSION ?? "0.0.0", - "node-version": process.version, - platform: process.platform, - arch: process.arch, +const resolveInstallationIdEffect = Effect.fn("Telemetry.resolveInstallationId")(function* () { + const configPath = getPrismaUserConfigPath(); + if (!configPath) return undefined; + const config = yield* readPrismaUserConfigEffect(configPath); + if (config.enableTelemetry === false) return undefined; + if (typeof config.installationId === "string" && config.installationId.length > 0) { + return config.installationId; + } + return yield* discloseAndMintEffect(configPath, config); }); +function getRuntime(): { name: string; version: string } { + if (process.versions.bun !== undefined) return { name: "bun", version: process.versions.bun }; + if (process.versions.deno !== undefined) return { name: "deno", version: process.versions.deno }; + return { name: "node", version: process.versions.node }; +} + +// `pnpm/10.27.0 npm/? node/v24.13.0 darwin arm64` → `pnpm/10.27.0` +function getInvokingPackageManager(userAgent = process.env.npm_config_user_agent): string | null { + const first = userAgent?.split(/\s+/)[0]; + return first?.includes("/") ? first : null; +} + +// Integration tests point the client at a local server, as they do for the Prisma CLI. +function getTelemetryEndpoint(override = process.env.PRISMA_TELEMETRY_ENDPOINT): string { + try { + return new URL("/events", override || TELEMETRY_BACKEND_URL).toString(); + } catch { + return new URL("/events", TELEMETRY_BACKEND_URL).toString(); + } +} + const sanitizeProperties = (properties: TelemetryProperties) => Object.fromEntries( Object.entries(properties).filter(([, value]) => value !== undefined), ) as Record>; export const trackCliTelemetryEffect = Effect.fn("Telemetry.track")(function* ( - event: string, - properties: TelemetryProperties, + event: TelemetryEvent, ) { - if (shouldDisableTelemetry()) return; - const distinctId = yield* getAnonymousIdEffect(); + if (isTelemetryDisabledByEnvironment()) return; + const installationId = yield* resolveInstallationIdEffect(); + if (!installationId) return; + const agent = yield* Effect.promise(() => + determineAgent().then( + (result) => (result.isAgent ? result.agent.name : null), + () => null, + ), + ); + const runtime = getRuntime(); yield* Effect.tryPromise(async (signal) => { - const response = await fetch(new URL("/i/v0/e/", TELEMETRY_HOST), { + const response = await fetch(getTelemetryEndpoint(), { method: "POST", headers: { "content-type": "application/json" }, redirect: "error", body: JSON.stringify({ - api_key: TELEMETRY_API_KEY, - distinct_id: distinctId, - event, - properties: sanitizeProperties({ - ...getCommonProperties(), - ...properties, - $process_person_profile: false, - $geoip_disable: true, - }), - timestamp: new Date().toISOString(), + installationId, + version: getCliVersion(), + command: event.command, + flags: event.flags, + runtimeName: runtime.name, + runtimeVersion: runtime.version, + os: process.platform, + arch: process.arch, + packageManager: getInvokingPackageManager(), + databaseTarget: event.databaseTarget, + tsVersion: null, + agent, + extensions: [], + exitCode: event.exitCode, + properties: sanitizeProperties(event.properties), }), signal, }); await response.body?.cancel(); }); }); - -export function trackCliTelemetry(event: string, properties: TelemetryProperties): Promise { - return applicationRuntime.runPromise( - trackCliTelemetryEffect(event, properties).pipe( - Effect.timeout(TELEMETRY_TIMEOUT_MS), - Effect.catch(() => Effect.void), - ), - ); -} diff --git a/src/telemetry/create.ts b/src/telemetry/create.ts index b21d0c6..cb1b5af 100644 --- a/src/telemetry/create.ts +++ b/src/telemetry/create.ts @@ -14,14 +14,34 @@ import { applicationRuntime } from "../runtime"; import { CommandExecutionError } from "../services/command-runner"; import { getPackageManagerErrorCode } from "../utils/package-manager-error-code"; -import { TELEMETRY_TIMEOUT_MS, trackCliTelemetryEffect } from "./client"; - -export const CREATE_PRISMA_NEXT_COMPLETED_EVENT = "cli:create_prisma_next_command_completed"; -export const CREATE_PRISMA_NEXT_FAILED_EVENT = "cli:create_prisma_next_command_failed"; -export const CREATE_PRISMA_NEXT_CANCELLED_EVENT = "cli:create_prisma_next_command_cancelled"; -export const PRISMA_INIT_COMPLETED_EVENT = "cli:create_prisma_orm_init_command_completed"; -export const PRISMA_INIT_FAILED_EVENT = "cli:create_prisma_orm_init_command_failed"; -export const PRISMA_INIT_CANCELLED_EVENT = "cli:create_prisma_orm_init_command_cancelled"; +import { TELEMETRY_TIMEOUT_MS, trackCliTelemetryEffect, type TelemetryProperties } from "./client"; + +// Every run reports under one command; `operation` and `outcome` tell runs apart. +export const CREATE_PRISMA_TELEMETRY_COMMAND = "create-prisma"; + +// A failed or cancelled run exits 1 (src/index.ts). +const exitCodes = { completed: 0, failed: 1, cancelled: 1 } as const; + +// Positional and `--name` values never leave the machine, so `name` reports no flag. +const flagNames = { + template: "template", + provider: "provider", + authoring: "authoring", + packageManager: "package-manager", + deploy: "deploy", + workspace: "workspace", + skills: "skills", + force: "force", + yes: "yes", + verbose: "verbose", + json: "json", +} as const satisfies Partial>; + +function getCreateFlags(input: CreateCommandInput): string[] { + return Object.entries(flagNames).flatMap(([key, flag]) => + input[key as keyof typeof flagNames] === undefined ? [] : [flag], + ); +} export type CreateTelemetryFailureStage = CreateFailureStage; @@ -57,22 +77,25 @@ function getTargetDirectoryState(context: CreateContext): string { return "non_empty_directory"; } +function getDatabaseTarget(input: CreateCommandInput, context?: CreateContext): string | null { + const setup = context?.operation === "orm_init" ? undefined : context?.prismaSetupContext; + return setup?.databaseProvider ?? input.provider ?? null; +} + function getBaseCreateProperties( input: CreateCommandInput, context?: CreateContext, -): Record { +): TelemetryProperties { const setup = context?.operation === "orm_init" ? undefined : context?.prismaSetupContext; const initializing = context?.operation === "orm_init"; return { - "telemetry-schema-version": 2, - command: "create", + "telemetry-schema-version": 3, operation: initializing ? "orm_init" : "create", "uses-defaults": input.yes === true || input.json === true, json: input.json === true, verbose: input.verbose === true, force: input.force === true, template: initializing ? null : (context?.template ?? input.template ?? null), - "database-provider": setup?.databaseProvider ?? input.provider ?? null, "authoring-style": setup?.authoring ?? input.authoring ?? null, "package-manager": initializing ? context.packageManager @@ -111,10 +134,15 @@ function getChildProcessFailureProperty(error: unknown): string | null { : null; } -function getPackageManagerErrorCodeProperty(error: unknown): string | null { - return error instanceof CommandExecutionError - ? (getPackageManagerErrorCode(error.command, error) ?? null) - : null; +// Our own install, or the one `prisma orm init` runs inside an existing app. +function getPackageManagerErrorCodeProperty( + error: unknown, + stage: CreateTelemetryFailureStage, +): string | null { + if (stage === "install_dependencies" && error instanceof CommandExecutionError) { + return getPackageManagerErrorCode(error.command, error) ?? null; + } + return error instanceof PrismaCliCommandError ? (error.packageManagerErrorCode ?? null) : null; } function getPrismaCliFailureProperty( @@ -134,20 +162,28 @@ function getPrismaCliCauseCodeProperty(error: unknown): string | null { return isStructuredErrorCode(causeCode) ? causeCode : null; } +const trackCreateOutcomeEffect = ( + outcome: keyof typeof exitCodes, + input: CreateCommandInput, + context: CreateContext | undefined, + properties: TelemetryProperties, +) => + trackCliTelemetryEffect({ + command: CREATE_PRISMA_TELEMETRY_COMMAND, + flags: getCreateFlags(input), + databaseTarget: getDatabaseTarget(input, context), + exitCode: exitCodes[outcome], + properties: { ...getBaseCreateProperties(input, context), outcome, ...properties }, + }).pipe( + Effect.timeout(TELEMETRY_TIMEOUT_MS), + Effect.catch(() => Effect.void), + ); + export const trackCreateCompletedEffect = Effect.fn("Telemetry.createCompleted")( function* (params: { input: CreateCommandInput; context: CreateContext; durationMs: number }) { - yield* trackCliTelemetryEffect( - params.context.operation === "orm_init" - ? PRISMA_INIT_COMPLETED_EVENT - : CREATE_PRISMA_NEXT_COMPLETED_EVENT, - { - ...getBaseCreateProperties(params.input, params.context), - "duration-ms": params.durationMs, - }, - ).pipe( - Effect.timeout(TELEMETRY_TIMEOUT_MS), - Effect.catch(() => Effect.void), - ); + yield* trackCreateOutcomeEffect("completed", params.input, params.context, { + "duration-ms": params.durationMs, + }); }, ); @@ -160,32 +196,20 @@ export const trackCreateFailedEffect = Effect.fn("Telemetry.createFailed")(funct reason: CreateFailureReason; targetDirectoryKind?: CreateTargetDirectoryKind; }) { - yield* trackCliTelemetryEffect( - params.context?.operation === "orm_init" - ? PRISMA_INIT_FAILED_EVENT - : CREATE_PRISMA_NEXT_FAILED_EVENT, - { - ...getBaseCreateProperties(params.input, params.context), - "duration-ms": params.durationMs, - "failure-class": getFailureClass(params.reason), - "failure-stage": params.stage, - "failure-reason": params.reason, - "target-directory-kind": params.targetDirectoryKind ?? null, - "error-name": getErrorName(params.error), - "error-code": getErrorCode(params.error), - "child-process-failure": getChildProcessFailureProperty(params.error), - "prisma-cli-command": getPrismaCliFailureProperty(params.error, "prismaCliCommand"), - "prisma-cli-error-code": getPrismaCliFailureProperty(params.error, "prismaCliErrorCode"), - "prisma-cli-cause-code": getPrismaCliCauseCodeProperty(params.error), - "package-manager-error-code": - params.stage === "install_dependencies" - ? getPackageManagerErrorCodeProperty(params.error) - : null, - }, - ).pipe( - Effect.timeout(TELEMETRY_TIMEOUT_MS), - Effect.catch(() => Effect.void), - ); + yield* trackCreateOutcomeEffect("failed", params.input, params.context, { + "duration-ms": params.durationMs, + "failure-class": getFailureClass(params.reason), + "failure-stage": params.stage, + "failure-reason": params.reason, + "target-directory-kind": params.targetDirectoryKind ?? null, + "error-name": getErrorName(params.error), + "error-code": getErrorCode(params.error), + "child-process-failure": getChildProcessFailureProperty(params.error), + "prisma-cli-command": getPrismaCliFailureProperty(params.error, "prismaCliCommand"), + "prisma-cli-error-code": getPrismaCliFailureProperty(params.error, "prismaCliErrorCode"), + "prisma-cli-cause-code": getPrismaCliCauseCodeProperty(params.error), + "package-manager-error-code": getPackageManagerErrorCodeProperty(params.error, params.stage), + }); }); export const trackCreateCancelledEffect = Effect.fn("Telemetry.createCancelled")( @@ -195,19 +219,10 @@ export const trackCreateCancelledEffect = Effect.fn("Telemetry.createCancelled") durationMs: number; stage: CreateCancellationStage; }) { - yield* trackCliTelemetryEffect( - params.context?.operation === "orm_init" - ? PRISMA_INIT_CANCELLED_EVENT - : CREATE_PRISMA_NEXT_CANCELLED_EVENT, - { - ...getBaseCreateProperties(params.input, params.context), - "duration-ms": params.durationMs, - "cancellation-stage": params.stage, - }, - ).pipe( - Effect.timeout(TELEMETRY_TIMEOUT_MS), - Effect.catch(() => Effect.void), - ); + yield* trackCreateOutcomeEffect("cancelled", params.input, params.context, { + "duration-ms": params.durationMs, + "cancellation-stage": params.stage, + }); }, ); diff --git a/tests/telemetry-client.test.ts b/tests/telemetry-client.test.ts index e9db9ae..f74d0f6 100644 --- a/tests/telemetry-client.test.ts +++ b/tests/telemetry-client.test.ts @@ -1,6 +1,6 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import { execa } from "execa"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -21,38 +21,51 @@ beforeAll(async () => { afterAll(async () => { if (bundleDir) await rm(bundleDir, { recursive: true, force: true }); }); -const anonymousId = "40f9096f-9b67-4de2-83ec-9d56d77f527b"; -const optOuts = [ - "DO_NOT_TRACK", - "CREATE_PRISMA_DISABLE_TELEMETRY", - "CREATE_PRISMA_TELEMETRY_DISABLED", - "CI", - "GITHUB_ACTIONS", -]; -async function sendTelemetry( - host: string, - env: Record = {}, - events = ["cli:create_prisma_next_command_completed"], -) { +let configHome: string; +beforeEach(async () => { + configHome = await mkdtemp(path.join(import.meta.dirname, ".telemetry-config-")); +}); +afterEach(async () => { + if (configHome) await rm(configHome, { recursive: true, force: true }); +}); + +const installationId = "40f9096f-9b67-4de2-83ec-9d56d77f527b"; +const configPath = () => path.join(configHome, "prisma", "config.json"); + +async function writeConfig(config: Record) { + await mkdir(path.dirname(configPath()), { recursive: true }); + await writeFile(configPath(), JSON.stringify(config)); +} + +const event = { + command: "create-prisma", + flags: ["template", "yes"], + databaseTarget: "postgres", + exitCode: 1, + properties: { + outcome: "failed", + "failure-stage": "composer_deploy", + "error-code": "ASSEMBLE.BUILD_FAILED", + optional: undefined, + }, +}; + +// A clean environment: the host's CI and coding-agent variables would otherwise decide +// whether the client sends and what it reports. +async function sendTelemetry(endpoint: string, env: Record = {}, runs = 1) { const result = await execa( "node", [ "--input-type=module", "--eval", ` - import { Effect, FileSystem } from "effect"; + import { NodeFileSystem } from "@effect/platform-node-shared"; + import { Effect } from "effect"; import { trackCliTelemetryEffect, TELEMETRY_TIMEOUT_MS } from ${JSON.stringify(pathToFileURL(clientPath).href)}; - const fs = FileSystem.makeNoop({ - readFileString: () => Effect.succeed(${JSON.stringify(JSON.stringify({ anonymousId }))}), - }); - for (const event of ${JSON.stringify(events)}) { - await Effect.runPromise(trackCliTelemetryEffect(event, { - "failure-stage": "composer_deploy", - "error-code": "ASSEMBLE.BUILD_FAILED", - optional: undefined, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), + for (let run = 0; run < ${runs}; run++) { + await Effect.runPromise(trackCliTelemetryEffect(${JSON.stringify(event)}).pipe( + Effect.provide(NodeFileSystem.layer), Effect.timeout(TELEMETRY_TIMEOUT_MS), Effect.catch(() => Effect.void), )); @@ -60,119 +73,160 @@ async function sendTelemetry( `, ], { + cwd: import.meta.dirname, + extendEnv: false, env: { - ...Object.fromEntries(optOuts.map((key) => [key, undefined])), - CREATE_PRISMA_TELEMETRY_API_KEY: "test-project-token", - CREATE_PRISMA_TELEMETRY_HOST: host, + PATH: process.env.PATH ?? "", + HOME: configHome, + XDG_CONFIG_HOME: configHome, + APPDATA: configHome, + PRISMA_TELEMETRY_ENDPOINT: endpoint, CREATE_PRISMA_CLI_VERSION: "test-version", + npm_config_user_agent: "pnpm/10.27.0 npm/? node/v24.13.0 darwin arm64", ...env, }, timeout: 5_000, }, ); expect(result.stdout).toBe(""); - expect(result.stderr).toBe(""); + return result.stderr; +} + +function startServer(onRequest: (request: Request) => Promise | Response) { + return Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: onRequest }); } describe("telemetry POST", () => { - test("sends one anonymous request per outcome with the existing properties and identity", async () => { - const requests: Array<{ - method: string; - path: string; - contentType: string | null; - body: { properties: Record; timestamp: string }; - }> = []; - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - requests.push({ - method: request.method, - path: new URL(request.url).pathname, - contentType: request.headers.get("content-type"), - body: await request.json(), - }); - return new Response(null, { status: 204 }); - }, + test("sends the Prisma CLI's event shape with create-prisma's outcome under its installation id", async () => { + await writeConfig({ installationId, otherSetting: true }); + const requests: Array<{ method: string; path: string; contentType: string | null; body: any }> = + []; + const server = startServer(async (request) => { + requests.push({ + method: request.method, + path: new URL(request.url).pathname, + contentType: request.headers.get("content-type"), + body: await request.json(), + }); + return new Response(null, { status: 202 }); }); - const events = [ - "cli:create_prisma_next_command_completed", - "cli:create_prisma_next_command_failed", - "cli:create_prisma_next_command_cancelled", - ]; try { - await sendTelemetry(server.url.toString(), {}, events); - expect(requests).toHaveLength(events.length); - for (const [index, request] of requests.entries()) { - expect(request).toMatchObject({ - method: "POST", - path: "/i/v0/e/", - contentType: "application/json", - body: { - api_key: "test-project-token", - distinct_id: anonymousId, - event: events[index], - properties: { - "cli-version": "test-version", - platform: process.platform, - arch: process.arch, - "failure-stage": "composer_deploy", - "error-code": "ASSEMBLE.BUILD_FAILED", - $process_person_profile: false, - $geoip_disable: true, - }, + expect(await sendTelemetry(server.url.toString())).toBe(""); + expect(requests).toHaveLength(1); + expect(requests[0]).toEqual({ + method: "POST", + path: "/events", + contentType: "application/json", + body: { + installationId, + version: "test-version", + command: "create-prisma", + flags: ["template", "yes"], + runtimeName: "node", + runtimeVersion: expect.any(String), + os: process.platform, + arch: process.arch, + packageManager: "pnpm/10.27.0", + databaseTarget: "postgres", + tsVersion: null, + agent: null, + extensions: [], + exitCode: 1, + properties: { + outcome: "failed", + "failure-stage": "composer_deploy", + "error-code": "ASSEMBLE.BUILD_FAILED", }, - }); - expect(request.body.properties).not.toHaveProperty("optional"); - expect(Number.isNaN(Date.parse(request.body.timestamp))).toBe(false); - } + }, + }); } finally { server.stop(true); } }); - test("does not send when opted out, in CI, or without a project token", async () => { + test("discloses once on the first run and stores the id beside the existing preferences", async () => { + await writeConfig({ otherSetting: true }); + const ids: string[] = []; + const server = startServer(async (request) => { + ids.push(((await request.json()) as { installationId: string }).installationId); + return new Response(null, { status: 202 }); + }); + try { + const stderr = await sendTelemetry(server.url.toString(), {}, 2); + expect(stderr.match(/Prisma collects anonymous CLI usage data/g)).toHaveLength(1); + expect(stderr).toContain("prisma telemetry disable"); + const stored = JSON.parse(await readFile(configPath(), "utf8")); + expect(stored).toEqual({ otherSetting: true, installationId: expect.any(String) }); + expect(ids).toEqual([stored.installationId, stored.installationId]); + expect(await sendTelemetry(server.url.toString())).toBe(""); + } finally { + server.stop(true); + } + }); + + test("does not send when opted out, opted out in the Prisma config, in CI, or from an unbuilt source", async () => { + await writeConfig({ installationId }); let requests = 0; - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - fetch() { - requests++; - return new Response(null, { status: 204 }); - }, + const server = startServer(() => { + requests++; + return new Response(null, { status: 202 }); }); try { - for (const key of optOuts) await sendTelemetry(server.url.toString(), { [key]: "1" }); - await sendTelemetry(server.url.toString(), { CREATE_PRISMA_TELEMETRY_API_KEY: "" }); + for (const env of [ + { PRISMA_DISABLE_TELEMETRY: "1" }, + { DO_NOT_TRACK: "1" }, + { CREATE_PRISMA_DISABLE_TELEMETRY: "" }, + { CREATE_PRISMA_TELEMETRY_DISABLED: "1" }, + { CI: "1" }, + { GITHUB_ACTIONS: "true" }, + { CREATE_PRISMA_CLI_VERSION: "0.0.0" }, + ]) { + await sendTelemetry(server.url.toString(), env); + } + await writeConfig({ installationId, enableTelemetry: false }); + expect(await sendTelemetry(server.url.toString())).toBe(""); expect(requests).toBe(0); } finally { server.stop(true); } }); + test("treats set-but-falsy Prisma opt-outs as not set, as the Prisma CLI does", async () => { + await writeConfig({ installationId }); + let requests = 0; + const server = startServer(() => { + requests++; + return new Response(null, { status: 202 }); + }); + try { + await sendTelemetry(server.url.toString(), { PRISMA_DISABLE_TELEMETRY: "0" }); + await sendTelemetry(server.url.toString(), { DO_NOT_TRACK: "false" }); + expect(requests).toBe(2); + } finally { + server.stop(true); + } + }); + test("aborts a stalled request and never retries or prints telemetry failures", async () => { + await writeConfig({ installationId }); let requests = 0; let aborted = false; - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - fetch(request) { - requests++; - request.signal.addEventListener("abort", () => { - aborted = true; - }); - return new Promise(() => {}); - }, + const server = startServer((request) => { + requests++; + request.signal.addEventListener("abort", () => { + aborted = true; + }); + return new Promise(() => {}); }); try { const started = performance.now(); - await sendTelemetry(server.url.toString()); + expect(await sendTelemetry(server.url.toString())).toBe(""); expect(performance.now() - started).toBeLessThan(3_500); expect(requests).toBe(1); expect(aborted).toBe(true); } finally { server.stop(true); } - await sendTelemetry("http://127.0.0.1:1"); + expect(await sendTelemetry("http://127.0.0.1:1")).toBe(""); }); }); diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts index 541983b..f100631 100644 --- a/tests/telemetry.test.ts +++ b/tests/telemetry.test.ts @@ -7,6 +7,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import type { CreatePromptContext, ExistingAppContext } from "../src/commands/create"; +import type { TelemetryEvent } from "../src/telemetry/client"; import type { CreateCommandInput } from "../src/types"; import { CommandExecutionError, CommandRunner } from "../src/services/command-runner"; import { runComposerDeployEffect } from "../src/tasks/composer/deploy-report"; @@ -16,25 +17,19 @@ import { getSpawnedCommandFailure, } from "../src/utils/child-process-failure"; -const trackCliTelemetry = mock(async () => {}); +// Records each event with its properties alongside, so tests can read either. +const trackCliTelemetry = mock( + async (_event: TelemetryEvent, _properties: TelemetryEvent["properties"]) => {}, +); mock.module("../src/telemetry/client", () => ({ TELEMETRY_TIMEOUT_MS: 2_000, - trackCliTelemetryEffect: (event: string, properties: Record) => - Effect.promise(() => trackCliTelemetry(event, properties)), + trackCliTelemetryEffect: (event: TelemetryEvent) => + Effect.promise(() => trackCliTelemetry(event, event.properties)), })); -const { - CREATE_PRISMA_NEXT_CANCELLED_EVENT, - CREATE_PRISMA_NEXT_COMPLETED_EVENT, - CREATE_PRISMA_NEXT_FAILED_EVENT, - PRISMA_INIT_COMPLETED_EVENT, - PRISMA_INIT_FAILED_EVENT, - PRISMA_INIT_CANCELLED_EVENT, - trackCreateCancelled, - trackCreateCompleted, - trackCreateFailed, -} = await import("../src/telemetry/create"); +const { trackCreateCancelled, trackCreateCompleted, trackCreateFailed } = + await import("../src/telemetry/create"); const createInput = { yes: true, name: "app" } satisfies CreateCommandInput; const createContext: CreatePromptContext = { @@ -88,7 +83,7 @@ async function trackFailure(error: unknown, stage: "install_dependencies" | "com stage, reason: stage === "composer_deploy" ? "composer_deploy_failed" : "dependency_install_failed", }); - return (trackCliTelemetry.mock.calls.at(-1) as unknown as [string, Record])[1]; + return trackCliTelemetry.mock.calls.at(-1)![1]; } describe("create telemetry", () => { @@ -110,11 +105,13 @@ describe("create telemetry", () => { reason: "prisma_init_failed", }); await trackCreateCancelled({ input, context, durationMs: 1, stage: "initialize_prisma" }); - const calls = trackCliTelemetry.mock.calls as Array<[string, Record]>; - expect(calls.map(([event]) => event)).toEqual([ - PRISMA_INIT_COMPLETED_EVENT, - PRISMA_INIT_FAILED_EVENT, - PRISMA_INIT_CANCELLED_EVENT, + const calls = trackCliTelemetry.mock.calls; + expect( + calls.map(([event]) => [event.command, event.properties.outcome, event.exitCode]), + ).toEqual([ + ["create-prisma", "completed", 0], + ["create-prisma", "failed", 1], + ["create-prisma", "cancelled", 1], ]); for (const [, properties] of calls) { expect(properties).toMatchObject({ @@ -129,17 +126,22 @@ describe("create telemetry", () => { test("tracks Composer deployment intent on completion", async () => { await trackCreateCompleted({ input: createInput, context: createContext, durationMs: 123 }); - expect(trackCliTelemetry).toHaveBeenCalledWith( - CREATE_PRISMA_NEXT_COMPLETED_EVENT, - expect.objectContaining({ - command: "create", - "telemetry-schema-version": 2, + const [event] = trackCliTelemetry.mock.calls[0]!; + expect(event).toMatchObject({ + command: "create-prisma", + flags: ["yes"], + databaseTarget: "postgres", + exitCode: 0, + properties: { + "telemetry-schema-version": 3, + operation: "create", + outcome: "completed", template: "hono", - "database-provider": "postgres", "should-deploy": true, "duration-ms": 123, - }), - ); + }, + }); + expect(event.properties).not.toHaveProperty("database-provider"); }); test("tracks a normalized setup failure without its raw message", async () => { @@ -151,11 +153,9 @@ describe("create telemetry", () => { stage: "plan_migration", reason: "migration_plan_failed", }); - const [event, properties] = trackCliTelemetry.mock.calls[0] as [ - string, - Record, - ]; - expect(event).toBe(CREATE_PRISMA_NEXT_FAILED_EVENT); + const [event, properties] = trackCliTelemetry.mock.calls[0]!; + expect(event).toMatchObject({ command: "create-prisma", exitCode: 1 }); + expect(properties.outcome).toBe("failed"); expect(properties).toEqual( expect.objectContaining({ "duration-ms": 456, @@ -185,7 +185,7 @@ describe("create telemetry", () => { reason, }); } - const calls = trackCliTelemetry.mock.calls as Array<[string, Record]>; + const calls = trackCliTelemetry.mock.calls; expect(calls).toHaveLength(5); expect(calls.map(([, properties]) => properties["failure-reason"])).toEqual([ "target_directory_not_empty", @@ -213,7 +213,7 @@ describe("create telemetry", () => { stage: "collect_context", reason: "invalid_project_name", }); - const calls = trackCliTelemetry.mock.calls as Array<[string, Record]>; + const calls = trackCliTelemetry.mock.calls; expect(calls.map(([, properties]) => properties["target-directory-kind"])).toEqual([ "existing_app", null, @@ -232,7 +232,7 @@ describe("create telemetry", () => { stage: "composer_deploy", reason: "composer_deploy_failed", }); - const [, properties] = trackCliTelemetry.mock.calls[0] as [string, Record]; + const [, properties] = trackCliTelemetry.mock.calls[0]!; expect(properties).toEqual( expect.objectContaining({ "prisma-cli-command": "app.deploy", @@ -276,10 +276,7 @@ describe("create telemetry", () => { reason: "dependency_install_failed", }); - const [, properties] = trackCliTelemetry.mock.calls.at(-1) as [ - string, - Record, - ]; + const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; expect(properties["child-process-failure"]).toBe(expectedFailure); expect(JSON.stringify(properties)).not.toContain("secret"); } @@ -347,10 +344,7 @@ describe("create telemetry", () => { stage: json ? "initialize_prisma" : "install_dependencies", reason: json ? "prisma_init_failed" : "dependency_install_failed", }); - const [, properties] = trackCliTelemetry.mock.calls.at(-1) as [ - string, - Record, - ]; + const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; expect(properties["child-process-failure"]).toBe(spec.failure); expect(properties["error-name"]).toBe( json ? "PrismaCliCommandError" : "CommandExecutionError", @@ -376,6 +370,57 @@ describe("create telemetry", () => { expect(JSON.stringify(properties)).not.toContain("registry"); }); + test("tracks the error code of the install prisma orm init runs inside an existing app", async () => { + const error = await Effect.runPromise( + runPrismaJsonCommandEffect({ + packageManager: "npm", + projectDir: process.cwd(), + args: ["orm", "init"], + }).pipe( + Effect.provideService(CommandRunner, { + run: () => + Effect.succeed({ + exitCode: 4, + stdout: JSON.stringify({ + ok: true, + result: { packagesInstalled: { status: "failed" } }, + diagnostics: [ + { + code: "CLI.INIT_INSTALL_FAILED", + severity: "error", + summary: "Failed to install dependencies", + why: "npm exited with code 1.", + meta: { + install: { + manager: "npm", + exitCode: 1, + stderrTail: + "npm error code EPERM\nnpm error path C:\\Users\\jane\\app\\node_modules", + }, + }, + }, + ], + }), + stderr: "", + }), + runChecked: () => Effect.die("unused"), + }), + Effect.flip, + ), + ); + await trackCreateFailed({ + input: createInput, + durationMs: 10, + error, + stage: "initialize_prisma", + reason: "prisma_init_failed", + }); + const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; + expect(properties["prisma-cli-error-code"]).toBe("CLI.INIT_INSTALL_FAILED"); + expect(properties["package-manager-error-code"]).toBe("EPERM"); + expect(JSON.stringify(properties)).not.toContain("jane"); + }); + test("tracks Composer's failure code from the deploy report", async () => { const { error, reportPath } = await failComposerDeploy( JSON.stringify({ @@ -408,14 +453,16 @@ describe("create telemetry", () => { durationMs: 789, stage, }); - expect(trackCliTelemetry).toHaveBeenCalledWith( - CREATE_PRISMA_NEXT_CANCELLED_EVENT, - expect.objectContaining({ + expect(trackCliTelemetry.mock.calls.at(-1)![0]).toMatchObject({ + command: "create-prisma", + exitCode: 1, + properties: { + outcome: "cancelled", "duration-ms": 789, "cancellation-stage": stage, "should-deploy": true, - }), - ); + }, + }); }, ); }); diff --git a/tsdown.config.ts b/tsdown.config.ts index 379e1ca..86bcfd5 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,8 +2,6 @@ import { defineConfig } from "tsdown"; import { readFileSync } from "node:fs"; const packageJson = JSON.parse(readFileSync(new URL("./package.json", import.meta.url), "utf8")); -const telemetryApiKey = process.env.CREATE_PRISMA_TELEMETRY_API_KEY ?? ""; -const telemetryHost = process.env.CREATE_PRISMA_TELEMETRY_HOST || "https://us.i.posthog.com"; export default defineConfig({ entry: ["src/index.ts", "src/cli.ts"], @@ -15,8 +13,6 @@ export default defineConfig({ outDir: "dist", env: { CREATE_PRISMA_CLI_VERSION: packageJson.version, - CREATE_PRISMA_TELEMETRY_API_KEY: telemetryApiKey, - CREATE_PRISMA_TELEMETRY_HOST: telemetryHost, }, outputOptions: { banner: "#!/usr/bin/env node", From 342620dad1bd24f9df69f741c34e74016b333aaa Mon Sep 17 00:00:00 2001 From: Aman Varshney Date: Fri, 16 Oct 2026 15:46:42 +0530 Subject: [PATCH 2/3] refactor(telemetry): send only the fields the Prisma CLI sends Drop the free-form properties so create-prisma reports flag names, never values, like the Prisma CLI. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/commands/create.ts | 33 +- src/create-outcome.ts | 2 - src/tasks/prisma-cli.ts | 19 -- src/telemetry/client.ts | 10 - src/telemetry/create.ts | 204 +----------- src/telemetry/index.ts | 7 - src/utils/package-manager-error-code.ts | 46 --- tests/package-manager-error-code.test.ts | 64 ---- tests/telemetry-client.test.ts | 13 +- tests/telemetry.test.ts | 407 +++-------------------- 11 files changed, 54 insertions(+), 753 deletions(-) delete mode 100644 src/telemetry/index.ts delete mode 100644 src/utils/package-manager-error-code.ts delete mode 100644 tests/package-manager-error-code.test.ts diff --git a/README.md b/README.md index e3c3a05..491ac38 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,4 @@ bun run build ## Telemetry -Published builds send anonymous usage telemetry to the same service as the Prisma CLI, and share its consent: the first run that reports prints a one-time notice, and `prisma telemetry disable` turns it off for both. Each run reports one event when it ends, under the installation id the Prisma CLI keeps in its user config, with the outcome, the chosen template, database, authoring style, package manager, the names of the flags you passed and how long the run took. It never includes project names, file paths, flag values, or database URLs. Failure events carry only stable identifiers such as exit codes and tool error codes, never messages or command output. A run rejected for a non-empty directory reports only whether that directory held an existing app, a Prisma project, or other files. Nothing is sent in CI. Disable it with `prisma telemetry disable`, `DO_NOT_TRACK=1`, or `PRISMA_DISABLE_TELEMETRY=1`; `CREATE_PRISMA_DISABLE_TELEMETRY` and `CREATE_PRISMA_TELEMETRY_DISABLED` still work. +Published builds send anonymous usage telemetry to the same service as the Prisma CLI, and share its consent: the first run that reports prints a one-time notice, and `prisma telemetry disable` turns it off for both. Each run reports one event when it ends, under the installation id the Prisma CLI keeps in its user config, with the same fields the Prisma CLI sends: the names of the flags you passed (never their values), the database, the exit code, and the runtime, OS and package manager. It never includes project names, file paths, or database URLs. Nothing is sent in CI. Disable it with `prisma telemetry disable`, `DO_NOT_TRACK=1`, or `PRISMA_DISABLE_TELEMETRY=1`; `CREATE_PRISMA_DISABLE_TELEMETRY` and `CREATE_PRISMA_TELEMETRY_DISABLED` still work. diff --git a/src/commands/create.ts b/src/commands/create.ts index 6a6465a..a67d151 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -1,5 +1,5 @@ import { cancel, intro, log, note, outro, spinner } from "@clack/prompts"; -import { Cause, Clock, Effect, Exit, Option, Ref, Schema } from "effect"; +import { Cause, Effect, Exit, Option, Ref, Schema } from "effect"; import { CreateCancellationError, CreateFailure } from "../create-outcome"; import { @@ -9,11 +9,7 @@ import { type CreateNextStep, } from "../result"; import { applicationRuntime } from "../runtime"; -import { - trackCreateCancelledEffect, - trackCreateCompletedEffect, - trackCreateFailedEffect, -} from "../telemetry/create"; +import { trackCreateEffect } from "../telemetry/create"; import { scaffoldCreateFrameworkTemplateEffect } from "../templates/render-create-template"; import { writeCreateTemplateDependenciesEffect } from "../tasks/install"; import { executePrismaSetupContextEffect } from "../tasks/setup-prisma"; @@ -222,18 +218,12 @@ const createProjectEffect = Effect.fn("Create.project")(function* ( export const runCreateCommandEffect = Effect.fn("Create.run")(function* ( rawInput: CreateCommandInput = {}, ) { - const startedAt = yield* Clock.currentTimeMillis; const inputRef = yield* Ref.make(rawInput); const contextRef = yield* Ref.make>(Option.none()); const exit = yield* Effect.exit(createProjectEffect(rawInput, inputRef, contextRef)); - const durationMs = (yield* Clock.currentTimeMillis) - startedAt; if (Exit.isSuccess(exit)) { - yield* trackCreateCompletedEffect({ - input: exit.value.input, - context: exit.value.context, - durationMs, - }); + yield* trackCreateEffect({ input: exit.value.input, context: exit.value.context, exitCode: 0 }); return exit.value.result; } @@ -241,12 +231,7 @@ export const runCreateCommandEffect = Effect.fn("Create.run")(function* ( const input = yield* Ref.get(inputRef); const context = Option.getOrUndefined(yield* Ref.get(contextRef)); if (error instanceof CreateCancellationError) { - yield* trackCreateCancelledEffect({ - input, - ...(context ? { context } : {}), - durationMs, - stage: error.stage, - }); + yield* trackCreateEffect({ input, ...(context ? { context } : {}), exitCode: 1 }); return createCommandFailureResult( error.stage, error.message ?? "Operation cancelled.", @@ -267,15 +252,7 @@ export const runCreateCommandEffect = Effect.fn("Create.run")(function* ( if (!failure.errorReported) { yield* Effect.sync(() => cancel(`Create command failed: ${failure.message}`, { output })); } - yield* trackCreateFailedEffect({ - input, - ...(context ? { context } : {}), - durationMs, - error: failure.cause ?? failure, - stage: failure.stage, - reason: failure.reason, - ...(failure.targetDirectoryKind ? { targetDirectoryKind: failure.targetDirectoryKind } : {}), - }); + yield* trackCreateEffect({ input, ...(context ? { context } : {}), exitCode: 1 }); return createCommandFailureResult( failure.stage, failure.message, diff --git a/src/create-outcome.ts b/src/create-outcome.ts index 350d564..442125e 100644 --- a/src/create-outcome.ts +++ b/src/create-outcome.ts @@ -116,8 +116,6 @@ export class PrismaCliCommandError extends Schema.TaggedError diagnostic.severity === "error"); const summary = failure?.summary ?? failure?.message; - const packageManagerErrorCode = getInstallErrorCode(failure); return yield* new PrismaCliCommandError({ message: redactSecrets( [summary, failure?.why].filter(Boolean).join(": ") || @@ -113,7 +95,6 @@ export const runPrismaJsonCommandEffect = Effect.fn("PrismaCli.runJson")(functio ? { command: envelope.commandId ?? envelope.command } : {}), ...(failure?.code ? { code: failure.code } : {}), - ...(packageManagerErrorCode ? { packageManagerErrorCode } : {}), stderr: redactSecrets(result.stderr), exitCode: result.exitCode, childProcessFailure: result.childProcessFailure, diff --git a/src/telemetry/client.ts b/src/telemetry/client.ts index 18e6e2a..fabcd10 100644 --- a/src/telemetry/client.ts +++ b/src/telemetry/client.ts @@ -10,15 +10,11 @@ const TELEMETRY_BACKEND_URL = "https://cmpbfbsdp09hr3jf7pojjs5qs.ewr.prisma.buil const TELEMETRY_DOCS_URL = "https://www.prisma.io/docs/cli"; export const TELEMETRY_TIMEOUT_MS = 2_000; -type TelemetryValue = boolean | number | string | string[] | null | undefined; -export type TelemetryProperties = Record; - export type TelemetryEvent = { command: string; flags: string[]; databaseTarget: string | null; exitCode: number; - properties: TelemetryProperties; }; type PrismaUserConfig = Record & { @@ -143,11 +139,6 @@ function getTelemetryEndpoint(override = process.env.PRISMA_TELEMETRY_ENDPOINT): } } -const sanitizeProperties = (properties: TelemetryProperties) => - Object.fromEntries( - Object.entries(properties).filter(([, value]) => value !== undefined), - ) as Record>; - export const trackCliTelemetryEffect = Effect.fn("Telemetry.track")(function* ( event: TelemetryEvent, ) { @@ -181,7 +172,6 @@ export const trackCliTelemetryEffect = Effect.fn("Telemetry.track")(function* ( agent, extensions: [], exitCode: event.exitCode, - properties: sanitizeProperties(event.properties), }), signal, }); diff --git a/src/telemetry/create.ts b/src/telemetry/create.ts index cb1b5af..fcc8197 100644 --- a/src/telemetry/create.ts +++ b/src/telemetry/create.ts @@ -1,28 +1,12 @@ import { Effect } from "effect"; import type { CreateContext } from "../commands/create"; -import { - isStructuredErrorCode, - PrismaCliCommandError, - type CreateCancellationStage, - type CreateFailureReason, - type CreateFailureStage, - type CreateTargetDirectoryKind, -} from "../create-outcome"; import type { CreateCommandInput } from "../types"; -import { applicationRuntime } from "../runtime"; -import { CommandExecutionError } from "../services/command-runner"; -import { getPackageManagerErrorCode } from "../utils/package-manager-error-code"; -import { TELEMETRY_TIMEOUT_MS, trackCliTelemetryEffect, type TelemetryProperties } from "./client"; +import { TELEMETRY_TIMEOUT_MS, trackCliTelemetryEffect } from "./client"; -// Every run reports under one command; `operation` and `outcome` tell runs apart. export const CREATE_PRISMA_TELEMETRY_COMMAND = "create-prisma"; -// A failed or cancelled run exits 1 (src/index.ts). -const exitCodes = { completed: 0, failed: 1, cancelled: 1 } as const; - -// Positional and `--name` values never leave the machine, so `name` reports no flag. const flagNames = { template: "template", provider: "provider", @@ -43,192 +27,22 @@ function getCreateFlags(input: CreateCommandInput): string[] { ); } -export type CreateTelemetryFailureStage = CreateFailureStage; - -const expectedRejectionReasons = new Set([ - "invalid_input", - "unsupported_node_version", - "unsupported_package_manager_version", - "package_manager_not_found", - "invalid_project_name", - "target_path_not_directory", - "target_directory_not_empty", - "target_has_migrations", - "unsupported_configuration", - "not_authenticated", - "workspace_missing", - "workspace_mismatch", - "project_name_collision", -]); - -function getFailureClass(reason: CreateFailureReason): "expected_rejection" | "technical_failure" { - return expectedRejectionReasons.has(reason) ? "expected_rejection" : "technical_failure"; -} - -function getTargetDirectoryState(context: CreateContext): string { - if (!context.targetPathState.exists) { - return "new"; - } - - if (context.targetPathState.isEmptyDirectory) { - return "empty_directory"; - } - - return "non_empty_directory"; -} - function getDatabaseTarget(input: CreateCommandInput, context?: CreateContext): string | null { const setup = context?.operation === "orm_init" ? undefined : context?.prismaSetupContext; return setup?.databaseProvider ?? input.provider ?? null; } -function getBaseCreateProperties( - input: CreateCommandInput, - context?: CreateContext, -): TelemetryProperties { - const setup = context?.operation === "orm_init" ? undefined : context?.prismaSetupContext; - const initializing = context?.operation === "orm_init"; - return { - "telemetry-schema-version": 3, - operation: initializing ? "orm_init" : "create", - "uses-defaults": input.yes === true || input.json === true, - json: input.json === true, - verbose: input.verbose === true, - force: input.force === true, - template: initializing ? null : (context?.template ?? input.template ?? null), - "authoring-style": setup?.authoring ?? input.authoring ?? null, - "package-manager": initializing - ? context.packageManager - : (setup?.packageManager ?? input.packageManager ?? null), - "agent-skills": initializing ? null : setup ? [...setup.skillAgents] : (input.skills ?? null), - "should-deploy": initializing ? false : (setup?.shouldDeploy ?? input.deploy ?? null), - "target-directory-state": context ? getTargetDirectoryState(context) : null, - }; -} - -function getErrorName(error: unknown): string | null { - if (error instanceof Error) { - return error.name; - } - - return error === undefined ? null : "UnknownError"; -} - -function getErrorCode(error: unknown): number | string | null { - if (typeof error !== "object" || error === null) { - return null; - } - - const exitCode = Reflect.get(error, "exitCode"); - if (typeof exitCode === "number") { - return exitCode; - } - - const code = Reflect.get(error, "code"); - return typeof code === "number" || typeof code === "string" ? code : null; -} - -function getChildProcessFailureProperty(error: unknown): string | null { - return error instanceof CommandExecutionError || error instanceof PrismaCliCommandError - ? (error.childProcessFailure ?? null) - : null; -} - -// Our own install, or the one `prisma orm init` runs inside an existing app. -function getPackageManagerErrorCodeProperty( - error: unknown, - stage: CreateTelemetryFailureStage, -): string | null { - if (stage === "install_dependencies" && error instanceof CommandExecutionError) { - return getPackageManagerErrorCode(error.command, error) ?? null; - } - return error instanceof PrismaCliCommandError ? (error.packageManagerErrorCode ?? null) : null; -} - -function getPrismaCliFailureProperty( - error: unknown, - property: "prismaCliCommand" | "prismaCliErrorCode" | "prismaCliCauseCode", -): string | null { - if (typeof error !== "object" || error === null) { - return null; - } - - const value = Reflect.get(error, property); - return typeof value === "string" && value.length > 0 ? value : null; -} - -function getPrismaCliCauseCodeProperty(error: unknown): string | null { - const causeCode = getPrismaCliFailureProperty(error, "prismaCliCauseCode"); - return isStructuredErrorCode(causeCode) ? causeCode : null; -} - -const trackCreateOutcomeEffect = ( - outcome: keyof typeof exitCodes, - input: CreateCommandInput, - context: CreateContext | undefined, - properties: TelemetryProperties, -) => +export const trackCreateEffect = (params: { + input: CreateCommandInput; + context?: CreateContext; + exitCode: number; +}) => trackCliTelemetryEffect({ command: CREATE_PRISMA_TELEMETRY_COMMAND, - flags: getCreateFlags(input), - databaseTarget: getDatabaseTarget(input, context), - exitCode: exitCodes[outcome], - properties: { ...getBaseCreateProperties(input, context), outcome, ...properties }, + flags: getCreateFlags(params.input), + databaseTarget: getDatabaseTarget(params.input, params.context), + exitCode: params.exitCode, }).pipe( Effect.timeout(TELEMETRY_TIMEOUT_MS), Effect.catch(() => Effect.void), ); - -export const trackCreateCompletedEffect = Effect.fn("Telemetry.createCompleted")( - function* (params: { input: CreateCommandInput; context: CreateContext; durationMs: number }) { - yield* trackCreateOutcomeEffect("completed", params.input, params.context, { - "duration-ms": params.durationMs, - }); - }, -); - -export const trackCreateFailedEffect = Effect.fn("Telemetry.createFailed")(function* (params: { - input: CreateCommandInput; - context?: CreateContext; - durationMs: number; - error?: unknown; - stage: CreateTelemetryFailureStage; - reason: CreateFailureReason; - targetDirectoryKind?: CreateTargetDirectoryKind; -}) { - yield* trackCreateOutcomeEffect("failed", params.input, params.context, { - "duration-ms": params.durationMs, - "failure-class": getFailureClass(params.reason), - "failure-stage": params.stage, - "failure-reason": params.reason, - "target-directory-kind": params.targetDirectoryKind ?? null, - "error-name": getErrorName(params.error), - "error-code": getErrorCode(params.error), - "child-process-failure": getChildProcessFailureProperty(params.error), - "prisma-cli-command": getPrismaCliFailureProperty(params.error, "prismaCliCommand"), - "prisma-cli-error-code": getPrismaCliFailureProperty(params.error, "prismaCliErrorCode"), - "prisma-cli-cause-code": getPrismaCliCauseCodeProperty(params.error), - "package-manager-error-code": getPackageManagerErrorCodeProperty(params.error, params.stage), - }); -}); - -export const trackCreateCancelledEffect = Effect.fn("Telemetry.createCancelled")( - function* (params: { - input: CreateCommandInput; - context?: CreateContext; - durationMs: number; - stage: CreateCancellationStage; - }) { - yield* trackCreateOutcomeEffect("cancelled", params.input, params.context, { - "duration-ms": params.durationMs, - "cancellation-stage": params.stage, - }); - }, -); - -export const trackCreateCompleted = (params: Parameters[0]) => - applicationRuntime.runPromise(trackCreateCompletedEffect(params)); -export const trackCreateFailed = (params: Parameters[0]) => - applicationRuntime.runPromise(trackCreateFailedEffect(params)); -export const trackCreateCancelled = (params: Parameters[0]) => - applicationRuntime.runPromise(trackCreateCancelledEffect(params)); diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts deleted file mode 100644 index ca6bb9d..0000000 --- a/src/telemetry/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { - trackCreateCancelled, - trackCreateCompleted, - trackCreateFailed, - type CreateTelemetryFailureStage, -} from "./create"; -export type { CreateCancellationStage, CreateFailureReason } from "../create-outcome"; diff --git a/src/utils/package-manager-error-code.ts b/src/utils/package-manager-error-code.ts deleted file mode 100644 index 95f389a..0000000 --- a/src/utils/package-manager-error-code.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { stripVTControlCharacters } from "node:util"; - -type ErrorCodeGrammar = { line: RegExp; allow: RegExp }; - -const grammars: Record = { - // npm error code E404 (npm <= 9: npm ERR! code E404) - npm: { - line: /^npm (?:error|ERR!) code (\S+)$/, - allow: /^E[A-Z0-9_]{2,40}$/, - }, - // [ERR_PNPM_FETCH_404] ... (pnpm <= 10 pads with thin spaces) - pnpm: { - line: /^(?:\[([A-Z0-9_]+)\] |\u2009([A-Z0-9_]+)\u2009 )/, - allow: /^(?:ERR_PNPM_[A-Z0-9_]{1,60}|E[A-Z0-9_]{2,40})$/, - }, - // ➤ YN0035: ...; YN0000 is the unnamed one - yarn: { - line: /^(?:➤ )?(YN\d{4}): /, - allow: /^YN(?!0000)\d{4}$/, - }, -}; - -function getGrammar(command: string): ErrorCodeGrammar | undefined { - const name = command - .replace(/^.*[\\/]/, "") - .replace(/\.(?:cmd|exe)$/i, "") - .toLowerCase(); - return Object.hasOwn(grammars, name) ? grammars[name] : undefined; -} - -// The result always matches the manager's allow-pattern, so it cannot carry free text. -export function getPackageManagerErrorCode( - command: string, - output: { stdout: string; stderr: string }, -): string | undefined { - const grammar = getGrammar(command); - if (!grammar) return undefined; - - const lines = stripVTControlCharacters(`${output.stdout}\n${output.stderr}`).split(/\r?\n/); - for (const line of lines.reverse()) { - const match = grammar.line.exec(line); - const candidate = match?.[1] ?? match?.[2]; - if (candidate !== undefined && grammar.allow.test(candidate)) return candidate; - } - return undefined; -} diff --git a/tests/package-manager-error-code.test.ts b/tests/package-manager-error-code.test.ts deleted file mode 100644 index ee82acf..0000000 --- a/tests/package-manager-error-code.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { getPackageManagerErrorCode } from "../src/utils/package-manager-error-code"; - -describe("getPackageManagerErrorCode", () => { - test.each([ - [ - "npm", - "npm error code ETARGET\nnpm error notarget No matching version found for left-pad@99.99.99.", - "ETARGET", - ], - [ - "npm", - "npm ERR! code ETARGET\nnpm ERR! notarget No matching version found for left-pad@99.99.99.", - "ETARGET", - ], - [ - "npm", - "npm error code 3\nnpm error path /Users/jane/my-app\nnpm error command failed", - undefined, - ], - [ - "pnpm", - "[ERR_PNPM_FETCH_404] GET https://registry.npmjs.org/nope: Not Found - 404", - "ERR_PNPM_FETCH_404", - ], - [ - "pnpm", - "\u2009ERR_PNPM_NO_MATCHING_VERSION\u2009 No matching version found for left-pad@99.99.99", - "ERR_PNPM_NO_MATCHING_VERSION", - ], - [ - "yarn", - "➤ YN0000: ┌ Resolution step\n➤ YN0082: │ left-pad@npm:99.99.99: No candidates found\n➤ YN0000: · Failed with errors in 0s 403ms", - "YN0082", - ], - [ - "bun", - 'error: No version matching "99.99.99" found for specifier "left-pad" (but package exists)', - undefined, - ], - ])("%s: %s", (command, output, expected) => { - expect(getPackageManagerErrorCode(command, { stdout: output, stderr: "" })).toBe(expected); - expect(getPackageManagerErrorCode(command, { stdout: "", stderr: output })).toBe(expected); - }); - - test("never returns free text from the identifier position", () => { - for (const secret of [ - "/Users/jane/my-app", - "https://jane:hunter2@registry.example.com/", - "npm_AbCdEf0123456789", - ]) { - for (const [command, output] of [ - ["npm", `npm error code ${secret}`], - ["pnpm", `[${secret}] failed`], - ["yarn", `➤ ${secret}: │ failed`], - ] as const) { - expect( - getPackageManagerErrorCode(command, { stdout: output, stderr: output }), - ).toBeUndefined(); - } - } - }); -}); diff --git a/tests/telemetry-client.test.ts b/tests/telemetry-client.test.ts index f74d0f6..fc8b905 100644 --- a/tests/telemetry-client.test.ts +++ b/tests/telemetry-client.test.ts @@ -43,12 +43,6 @@ const event = { flags: ["template", "yes"], databaseTarget: "postgres", exitCode: 1, - properties: { - outcome: "failed", - "failure-stage": "composer_deploy", - "error-code": "ASSEMBLE.BUILD_FAILED", - optional: undefined, - }, }; // A clean environment: the host's CI and coding-agent variables would otherwise decide @@ -97,7 +91,7 @@ function startServer(onRequest: (request: Request) => Promise | Respon } describe("telemetry POST", () => { - test("sends the Prisma CLI's event shape with create-prisma's outcome under its installation id", async () => { + test("sends the Prisma CLI's event shape under its installation id", async () => { await writeConfig({ installationId, otherSetting: true }); const requests: Array<{ method: string; path: string; contentType: string | null; body: any }> = []; @@ -132,11 +126,6 @@ describe("telemetry POST", () => { agent: null, extensions: [], exitCode: 1, - properties: { - outcome: "failed", - "failure-stage": "composer_deploy", - "error-code": "ASSEMBLE.BUILD_FAILED", - }, }, }); } finally { diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts index f100631..f08e9fb 100644 --- a/tests/telemetry.test.ts +++ b/tests/telemetry.test.ts @@ -1,37 +1,29 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; -import { NodeFileSystem } from "@effect/platform-node-shared"; import { Effect } from "effect"; -import { existsSync, writeFileSync } from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import type { CreatePromptContext, ExistingAppContext } from "../src/commands/create"; +import type { CreatePromptContext } from "../src/commands/create"; +import { PrismaCliCommandError } from "../src/create-outcome"; import type { TelemetryEvent } from "../src/telemetry/client"; import type { CreateCommandInput } from "../src/types"; import { CommandExecutionError, CommandRunner } from "../src/services/command-runner"; -import { runComposerDeployEffect } from "../src/tasks/composer/deploy-report"; import { runPrismaJsonCommandEffect } from "../src/tasks/prisma-cli"; -import { - getChildProcessFailure, - getSpawnedCommandFailure, -} from "../src/utils/child-process-failure"; +import { getSpawnedCommandFailure } from "../src/utils/child-process-failure"; -// Records each event with its properties alongside, so tests can read either. -const trackCliTelemetry = mock( - async (_event: TelemetryEvent, _properties: TelemetryEvent["properties"]) => {}, -); +const trackCliTelemetry = mock(async (_event: TelemetryEvent) => {}); mock.module("../src/telemetry/client", () => ({ TELEMETRY_TIMEOUT_MS: 2_000, trackCliTelemetryEffect: (event: TelemetryEvent) => - Effect.promise(() => trackCliTelemetry(event, event.properties)), + Effect.promise(() => trackCliTelemetry(event)), })); -const { trackCreateCancelled, trackCreateCompleted, trackCreateFailed } = - await import("../src/telemetry/create"); +const { trackCreateEffect } = await import("../src/telemetry/create"); +const trackCreate = (params: Parameters[0]) => + Effect.runPromise(trackCreateEffect(params)); -const createInput = { yes: true, name: "app" } satisfies CreateCommandInput; const createContext: CreatePromptContext = { targetDirectory: "/tmp/app", targetPathState: { exists: false, isDirectory: true, isEmptyDirectory: true }, @@ -44,244 +36,44 @@ const createContext: CreatePromptContext = { databaseProvider: "postgres", authoring: "psl", packageManager: "bun", - skillAgents: ["claude", "cursor", "agents", "devin"], - shouldDeploy: true, + skillAgents: ["claude"], + shouldDeploy: false, shouldPromptForWorkspace: false, }, }; beforeEach(() => trackCliTelemetry.mockClear()); -async function failComposerDeploy(report?: string) { - let reportPath = ""; - const run = (spec: { args: readonly string[] }) => - Effect.sync(() => { - reportPath = spec.args[spec.args.indexOf("--report") + 1]!; - if (report !== undefined) writeFileSync(reportPath, report); - return { - exitCode: 1, - stdout: '{"ok":false,"commandId":"deploy","error":{"code":"CLI.CHILD_PROCESS_FAILED"}}', - stderr: "", - }; - }); - const error = await Effect.runPromise( - runComposerDeployEffect({ packageManager: "npm", projectDir: process.cwd() }).pipe( - Effect.provideService(CommandRunner, { run, runChecked: run }), - Effect.provide(NodeFileSystem.layer), - Effect.flip, - ), - ); - return { error, reportPath }; -} - -async function trackFailure(error: unknown, stage: "install_dependencies" | "composer_deploy") { - await trackCreateFailed({ - input: createInput, - context: createContext, - durationMs: 10, - error, - stage, - reason: stage === "composer_deploy" ? "composer_deploy_failed" : "dependency_install_failed", - }); - return trackCliTelemetry.mock.calls.at(-1)![1]; -} - describe("create telemetry", () => { - test("keeps ORM initialization outcomes separate from starter and deployment metrics", async () => { - const context: ExistingAppContext = { - operation: "orm_init", - targetDirectory: "/tmp/existing", - targetPathState: { exists: true, isDirectory: true, isEmptyDirectory: false }, - projectPackageName: "existing", - packageManager: "bun", - }; - const input = { ...createInput, template: "next" as const, deploy: true }; - await trackCreateCompleted({ input, context, durationMs: 1 }); - await trackCreateFailed({ - input, - context, - durationMs: 1, - stage: "initialize_prisma", - reason: "prisma_init_failed", - }); - await trackCreateCancelled({ input, context, durationMs: 1, stage: "initialize_prisma" }); - const calls = trackCliTelemetry.mock.calls; - expect( - calls.map(([event]) => [event.command, event.properties.outcome, event.exitCode]), - ).toEqual([ - ["create-prisma", "completed", 0], - ["create-prisma", "failed", 1], - ["create-prisma", "cancelled", 1], - ]); - for (const [, properties] of calls) { - expect(properties).toMatchObject({ - operation: "orm_init", - template: null, - "should-deploy": false, - "agent-skills": null, - }); - expect(JSON.stringify(properties)).not.toContain("/tmp/existing"); - } - }); - - test("tracks Composer deployment intent on completion", async () => { - await trackCreateCompleted({ input: createInput, context: createContext, durationMs: 123 }); - const [event] = trackCliTelemetry.mock.calls[0]!; - expect(event).toMatchObject({ - command: "create-prisma", - flags: ["yes"], - databaseTarget: "postgres", - exitCode: 0, - properties: { - "telemetry-schema-version": 3, - operation: "create", - outcome: "completed", - template: "hono", - "should-deploy": true, - "duration-ms": 123, - }, - }); - expect(event.properties).not.toHaveProperty("database-provider"); - }); - - test("tracks a normalized setup failure without its raw message", async () => { - await trackCreateFailed({ - input: createInput, - context: createContext, - durationMs: 456, - error: Object.assign(new Error("DATABASE_URL=secret"), { code: "ERR_TEST" }), - stage: "plan_migration", - reason: "migration_plan_failed", - }); - const [event, properties] = trackCliTelemetry.mock.calls[0]!; - expect(event).toMatchObject({ command: "create-prisma", exitCode: 1 }); - expect(properties.outcome).toBe("failed"); - expect(properties).toEqual( - expect.objectContaining({ - "duration-ms": 456, - "failure-class": "technical_failure", - "error-code": "ERR_TEST", - "failure-stage": "plan_migration", - "failure-reason": "migration_plan_failed", - }), - ); - expect(properties).not.toHaveProperty("error-message"); - expect(JSON.stringify(properties)).not.toContain("secret"); - }); - - test("separates expected input and environment rejections from technical failures", async () => { - for (const reason of [ - "target_directory_not_empty", - "target_has_migrations", - "workspace_missing", - "unsupported_package_manager_version", - "package_manager_not_found", - ] as const) { - await trackCreateFailed({ - input: createInput, - context: createContext, - durationMs: 10, - stage: reason === "workspace_missing" ? "select_workspace" : "collect_context", - reason, - }); - } - const calls = trackCliTelemetry.mock.calls; - expect(calls).toHaveLength(5); - expect(calls.map(([, properties]) => properties["failure-reason"])).toEqual([ - "target_directory_not_empty", - "target_has_migrations", - "workspace_missing", - "unsupported_package_manager_version", - "package_manager_not_found", - ]); - for (const [, properties] of calls) { - expect(properties["failure-class"]).toBe("expected_rejection"); - } - }); - - test("reports what a rejected non-empty directory held, and nothing more", async () => { - await trackCreateFailed({ - input: createInput, - durationMs: 10, - stage: "collect_context", - reason: "target_directory_not_empty", - targetDirectoryKind: "existing_app", - }); - await trackCreateFailed({ - input: createInput, - durationMs: 10, - stage: "collect_context", - reason: "invalid_project_name", - }); - const calls = trackCliTelemetry.mock.calls; - expect(calls.map(([, properties]) => properties["target-directory-kind"])).toEqual([ - "existing_app", - null, + test("reports flag names, the database target and the exit code, never values", async () => { + const input = { + name: "secret-app", + template: "hono", + provider: "postgres", + yes: true, + } satisfies CreateCommandInput; + await trackCreate({ input, context: createContext, exitCode: 0 }); + await trackCreate({ input, exitCode: 1 }); + expect(trackCliTelemetry.mock.calls).toEqual([ + [ + { + command: "create-prisma", + flags: ["template", "provider", "yes"], + databaseTarget: "postgres", + exitCode: 0, + }, + ], + [ + { + command: "create-prisma", + flags: ["template", "provider", "yes"], + databaseTarget: "postgres", + exitCode: 1, + }, + ], ]); }); - test("tracks stable Prisma CLI failure fields without raw output", async () => { - await trackCreateFailed({ - input: createInput, - context: createContext, - durationMs: 456, - error: Object.assign(new Error("token=secret"), { - prismaCliCommand: "app.deploy", - prismaCliErrorCode: "APP.DEPLOY_FAILED", - }), - stage: "composer_deploy", - reason: "composer_deploy_failed", - }); - const [, properties] = trackCliTelemetry.mock.calls[0]!; - expect(properties).toEqual( - expect.objectContaining({ - "prisma-cli-command": "app.deploy", - "prisma-cli-error-code": "APP.DEPLOY_FAILED", - }), - ); - expect(JSON.stringify(properties)).not.toContain("secret"); - }); - - test("classifies child-process failures without capturing command output", async () => { - const cases = [ - [{ timedOut: true }, "timed_out"], - [{ isCanceled: true }, "cancelled"], - [{ isMaxBuffer: true }, "max_buffer"], - [{ signal: "SIGINT", isTerminated: true }, "interrupted"], - [{ exitCode: 0xc000013a }, "interrupted"], - [{ signal: "SIGTERM", isTerminated: true }, "terminated"], - [{ code: "ENOENT" }, "command_not_found"], - [{ code: "EACCES" }, "permission_denied"], - [{ exitCode: 1 }, "non_zero_exit"], - [{ code: "UNKNOWN" }, "spawn_failed"], - ] as const; - - for (const [details, expectedFailure] of cases) { - await trackCreateFailed({ - input: createInput, - context: createContext, - durationMs: 10, - error: new CommandExecutionError({ - command: "secret-command", - args: ["secret-argument"], - stdout: "token=secret", - stderr: "token=secret", - childProcessFailure: getChildProcessFailure({ - name: "ExecaError", - failed: true, - ...details, - }), - }), - stage: "install_dependencies", - reason: "dependency_install_failed", - }); - - const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; - expect(properties["child-process-failure"]).toBe(expectedFailure); - expect(JSON.stringify(properties)).not.toContain("secret"); - } - }); - test("recognizes a missing Windows command that cmd.exe reports as a failed exit", async () => { const binDirectory = await mkdtemp(path.join(tmpdir(), "create-prisma-bin-")); try { @@ -337,132 +129,9 @@ describe("create telemetry", () => { : runner.runChecked(command); }).pipe(Effect.provide(CommandRunner.layer), Effect.flip), ); - await trackCreateFailed({ - input: createInput, - durationMs: 10, - error, - stage: json ? "initialize_prisma" : "install_dependencies", - reason: json ? "prisma_init_failed" : "dependency_install_failed", - }); - const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; - expect(properties["child-process-failure"]).toBe(spec.failure); - expect(properties["error-name"]).toBe( - json ? "PrismaCliCommandError" : "CommandExecutionError", - ); - expect(JSON.stringify(properties)).not.toContain("secret"); + expect(error).toBeInstanceOf(json ? PrismaCliCommandError : CommandExecutionError); + expect(error.childProcessFailure).toBe(spec.failure); } } }); - - test("tracks the package manager's error code for install failures", async () => { - const properties = await trackFailure( - new CommandExecutionError({ - command: "npm", - args: ["install"], - exitCode: 1, - stdout: "", - stderr: - "npm error code E404\nnpm error 404 Not Found - GET https://registry.npmjs.org/nope", - }), - "install_dependencies", - ); - expect(properties["package-manager-error-code"]).toBe("E404"); - expect(JSON.stringify(properties)).not.toContain("registry"); - }); - - test("tracks the error code of the install prisma orm init runs inside an existing app", async () => { - const error = await Effect.runPromise( - runPrismaJsonCommandEffect({ - packageManager: "npm", - projectDir: process.cwd(), - args: ["orm", "init"], - }).pipe( - Effect.provideService(CommandRunner, { - run: () => - Effect.succeed({ - exitCode: 4, - stdout: JSON.stringify({ - ok: true, - result: { packagesInstalled: { status: "failed" } }, - diagnostics: [ - { - code: "CLI.INIT_INSTALL_FAILED", - severity: "error", - summary: "Failed to install dependencies", - why: "npm exited with code 1.", - meta: { - install: { - manager: "npm", - exitCode: 1, - stderrTail: - "npm error code EPERM\nnpm error path C:\\Users\\jane\\app\\node_modules", - }, - }, - }, - ], - }), - stderr: "", - }), - runChecked: () => Effect.die("unused"), - }), - Effect.flip, - ), - ); - await trackCreateFailed({ - input: createInput, - durationMs: 10, - error, - stage: "initialize_prisma", - reason: "prisma_init_failed", - }); - const [, properties] = trackCliTelemetry.mock.calls.at(-1)!; - expect(properties["prisma-cli-error-code"]).toBe("CLI.INIT_INSTALL_FAILED"); - expect(properties["package-manager-error-code"]).toBe("EPERM"); - expect(JSON.stringify(properties)).not.toContain("jane"); - }); - - test("tracks Composer's failure code from the deploy report", async () => { - const { error, reportPath } = await failComposerDeploy( - JSON.stringify({ - version: 1, - failure: { code: "DEPLOY.ENGINE_FAILED", message: "failed in /Users/jane/my-app" }, - }), - ); - const properties = await trackFailure(error, "composer_deploy"); - expect(properties["prisma-cli-error-code"]).toBe("CLI.CHILD_PROCESS_FAILED"); - expect(properties["prisma-cli-cause-code"]).toBe("DEPLOY.ENGINE_FAILED"); - expect(JSON.stringify(properties)).not.toContain("jane"); - expect(existsSync(path.dirname(reportPath))).toBe(false); - }); - - test("keeps the original deploy error when the report is missing or invalid", async () => { - for (const report of [undefined, "not json"]) { - const { error } = await failComposerDeploy(report); - expect(error).toMatchObject({ code: "CLI.CHILD_PROCESS_FAILED", exitCode: 1 }); - expect(error).not.toHaveProperty("causeCode"); - expect((await trackFailure(error, "composer_deploy"))["prisma-cli-cause-code"]).toBeNull(); - } - }); - - test.each(["select_workspace", "authenticate"] as const)( - "tracks %s cancellation as a separate outcome", - async (stage) => { - await trackCreateCancelled({ - input: createInput, - context: createContext, - durationMs: 789, - stage, - }); - expect(trackCliTelemetry.mock.calls.at(-1)![0]).toMatchObject({ - command: "create-prisma", - exitCode: 1, - properties: { - outcome: "cancelled", - "duration-ms": 789, - "cancellation-stage": stage, - "should-deploy": true, - }, - }); - }, - ); }); From 0a1e2466d1019af549aa4925a932b3ad9aa85dcf Mon Sep 17 00:00:00 2001 From: Aman Varshney Date: Tue, 17 Nov 2026 13:59:13 +0530 Subject: [PATCH 3/3] fix(telemetry): drop the deleted test from test:unit and list every reported field in the README --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 491ac38..b7f9905 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,4 @@ bun run build ## Telemetry -Published builds send anonymous usage telemetry to the same service as the Prisma CLI, and share its consent: the first run that reports prints a one-time notice, and `prisma telemetry disable` turns it off for both. Each run reports one event when it ends, under the installation id the Prisma CLI keeps in its user config, with the same fields the Prisma CLI sends: the names of the flags you passed (never their values), the database, the exit code, and the runtime, OS and package manager. It never includes project names, file paths, or database URLs. Nothing is sent in CI. Disable it with `prisma telemetry disable`, `DO_NOT_TRACK=1`, or `PRISMA_DISABLE_TELEMETRY=1`; `CREATE_PRISMA_DISABLE_TELEMETRY` and `CREATE_PRISMA_TELEMETRY_DISABLED` still work. +Published builds send anonymous usage telemetry to the same service as the Prisma CLI, and share its consent: the first run that reports prints a one-time notice, and `prisma telemetry disable` turns it off for both. Each run reports one event when it ends, under the installation id the Prisma CLI keeps in its user config, with the same fields the Prisma CLI sends: the names of the flags you passed (never their values), the database, the exit code, the create-prisma version, the runtime and its version, the OS and CPU architecture, the package manager, and the coding agent running it, if one is detected. It never includes project names, file paths, or database URLs. Nothing is sent in CI. Disable it with `prisma telemetry disable`, `DO_NOT_TRACK=1`, or `PRISMA_DISABLE_TELEMETRY=1`; `CREATE_PRISMA_DISABLE_TELEMETRY` and `CREATE_PRISMA_TELEMETRY_DISABLED` still work. diff --git a/package.json b/package.json index bec81da..88b6ad7 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "dev": "tsdown --watch", "start": "bun run ./dist/cli.mjs", "test": "bun run test:unit && bun run test:e2e", - "test:unit": "bun test ./tests/composer-dev-bun.test.ts ./tests/create-context.test.ts ./tests/dependencies.test.ts ./tests/deploy-with-composer.test.ts ./tests/initialize-git.test.ts ./tests/install.test.ts ./tests/json-output.test.ts ./tests/node-version.test.ts ./tests/package-manager-error-code.test.ts ./tests/pin-problems.test.ts ./tests/setup-prisma.test.ts ./tests/telemetry-client.test.ts ./tests/telemetry.test.ts", + "test:unit": "bun test ./tests/composer-dev-bun.test.ts ./tests/create-context.test.ts ./tests/dependencies.test.ts ./tests/deploy-with-composer.test.ts ./tests/initialize-git.test.ts ./tests/install.test.ts ./tests/json-output.test.ts ./tests/node-version.test.ts ./tests/pin-problems.test.ts ./tests/setup-prisma.test.ts ./tests/telemetry-client.test.ts ./tests/telemetry.test.ts", "test:e2e": "bun test --timeout 180000 ./tests/e2e/create-prisma.e2e.test.ts", "check": "bun run format:check && bun run lint", "lint": "oxlint . --deny-warnings",