diff --git a/.config/cspell/cspell.json b/.config/cspell/cspell.json index 6e84fb7..53021fd 100644 --- a/.config/cspell/cspell.json +++ b/.config/cspell/cspell.json @@ -40,6 +40,7 @@ "GLIBCXX", "libstdc", "noproxy", - "pgrep" + "pgrep", + "varint" ] } diff --git a/.github/workflows/github-cli.yml b/.github/workflows/github-cli.yml new file mode 100644 index 0000000..48f6179 --- /dev/null +++ b/.github/workflows/github-cli.yml @@ -0,0 +1,27 @@ +name: "GitHub CLI" + +on: + pull_request: + paths: + - "features/github-cli/config.json" + push: + branches: + - main + paths: + - "features/github-cli/config.json" + workflow_dispatch: + +concurrency: + group: "github-cli-${{ github.ref }}" + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + verify: + uses: ./.github/workflows/verify.yml + permissions: + contents: write + packages: write + with: + feature: github-cli + binaries: false + scenarios: '["default"]' diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml index 00ca32c..f8043b9 100644 --- a/.github/workflows/renovate.yml +++ b/.github/workflows/renovate.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - feature: [dotnet, ripgrep, code-server, uv] + feature: [dotnet, ripgrep, code-server, uv, github-cli] steps: - name: Checkout (GitHub) uses: actions/checkout@v7 diff --git a/features/github-cli/Dockerfile b/features/github-cli/Dockerfile new file mode 100644 index 0000000..7e95919 --- /dev/null +++ b/features/github-cli/Dockerfile @@ -0,0 +1,9 @@ +FROM scratch AS linux-amd64-base +FROM scratch AS linux-arm64-base + +ARG TARGETOS +ARG TARGETARCH +FROM ${TARGETOS}-${TARGETARCH}-base +ARG TARGETOS +ARG TARGETARCH +COPY ${TARGETOS}/${TARGETARCH}/features/github-cli/ /features/github-cli/ diff --git a/features/github-cli/NOTES.md b/features/github-cli/NOTES.md new file mode 100644 index 0000000..4c93823 --- /dev/null +++ b/features/github-cli/NOTES.md @@ -0,0 +1,11 @@ +## Installed files + +| Source | Destination | +| --------------------------- | ---------------------------------------------------- | +| `bin/gh` | `/usr/local/bin/gh` | +| `share/man/man1/*.1` | `/usr/local/share/man/man1/` | +| `LICENSE` | `/usr/local/share/doc/gh/LICENSE` | +| `gh completion -s bash` | `/usr/local/share/bash-completion/completions/gh` | +| `gh completion -s zsh` | `/usr/local/share/zsh/site-functions/_gh` | +| `gh completion -s fish` | `/usr/local/share/fish/vendor_completions.d/gh.fish` | +| `export GH_TELEMETRY=false` | `/etc/profile.d/gh.sh` | diff --git a/features/github-cli/README.md b/features/github-cli/README.md new file mode 100644 index 0000000..ee07b0e --- /dev/null +++ b/features/github-cli/README.md @@ -0,0 +1,28 @@ +# GitHub CLI (github-cli) + +Installs the GitHub CLI (gh). + +## Example Usage + +```dockerfile +FROM ghcr.io/devcontainer-config/features/github-cli AS github-cli + +FROM base-image +COPY --from=github-cli /features/github-cli/ /opt/devcontainer-config/features/github-cli/ +``` + +## Installed files + +| Source | Destination | +| --------------------------- | ---------------------------------------------------- | +| `bin/gh` | `/usr/local/bin/gh` | +| `share/man/man1/*.1` | `/usr/local/share/man/man1/` | +| `LICENSE` | `/usr/local/share/doc/gh/LICENSE` | +| `gh completion -s bash` | `/usr/local/share/bash-completion/completions/gh` | +| `gh completion -s zsh` | `/usr/local/share/zsh/site-functions/_gh` | +| `gh completion -s fish` | `/usr/local/share/fish/vendor_completions.d/gh.fish` | +| `export GH_TELEMETRY=false` | `/etc/profile.d/gh.sh` | + +--- + +_Note: This file was auto-generated from the [feature.json](https://github.com/devcontainer-config/features/blob/main/features/github-cli/feature.json). Add additional notes to a `NOTES.md`._ diff --git a/features/github-cli/config.json b/features/github-cli/config.json new file mode 100644 index 0000000..2a46620 --- /dev/null +++ b/features/github-cli/config.json @@ -0,0 +1,18 @@ +{ + "revision": 1, + "versions": { + "2.104.0": { + "linux-x64": { + "url": "https://github.com/cli/cli/releases/download/v2.104.0/gh_2.104.0_linux_amd64.tar.gz", + "sha256": "98cb101ad87dbbd3f6b2f2bfcbd43197ccd07643140121d8a9cfaa91eadbc24e" + }, + "linux-arm64": { + "url": "https://github.com/cli/cli/releases/download/v2.104.0/gh_2.104.0_linux_arm64.tar.gz", + "sha256": "cb34cefc6f4e77b90731374a2b737516bb8f3033c6ca46aa949e5344f68069e3" + } + } + }, + "aliases": { + "2.104.0": ["2.104", "2", "latest"] + } +} diff --git a/features/github-cli/feature.json b/features/github-cli/feature.json new file mode 100644 index 0000000..31a2c48 --- /dev/null +++ b/features/github-cli/feature.json @@ -0,0 +1,6 @@ +{ + "id": "github-cli", + "name": "GitHub CLI", + "description": "Installs the GitHub CLI (gh).", + "image": "ghcr.io/devcontainer-config/features/github-cli" +} diff --git a/features/github-cli/install b/features/github-cli/install new file mode 100755 index 0000000..a6a1d7e --- /dev/null +++ b/features/github-cli/install @@ -0,0 +1,37 @@ +#!/bin/sh +set -e + +mkdir -p /usr/local/bin +ln -sfn "$PWD/bin/gh" /usr/local/bin/gh + +mkdir -p /usr/local/share/man/man1 +cp -p share/man/man1/*.1 /usr/local/share/man/man1/ + +mkdir -p /usr/local/share/doc/gh +cp -p LICENSE /usr/local/share/doc/gh/LICENSE + +temp=$(mktemp -d) +trap 'rm -rf "$temp"' EXIT +mkdir -p "$temp/home" "$temp/config" "$temp/state" "$temp/data" "$temp/cache" + +completion() { + GH_TELEMETRY=false \ + HOME="$temp/home" \ + XDG_CONFIG_HOME="$temp/config" \ + XDG_STATE_HOME="$temp/state" \ + XDG_DATA_HOME="$temp/data" \ + XDG_CACHE_HOME="$temp/cache" \ + "$PWD/bin/gh" completion -s "$1" +} + +mkdir -p /usr/local/share/bash-completion/completions +completion bash > /usr/local/share/bash-completion/completions/gh + +mkdir -p /usr/local/share/zsh/site-functions +completion zsh > /usr/local/share/zsh/site-functions/_gh + +mkdir -p /usr/local/share/fish/vendor_completions.d +completion fish > /usr/local/share/fish/vendor_completions.d/gh.fish + +mkdir -p /etc/profile.d +printf '%s\n' 'export GH_TELEMETRY=false' > /etc/profile.d/gh.sh diff --git a/scripts/tasks/features/attestations.ts b/scripts/tasks/features/attestations.ts new file mode 100644 index 0000000..5474767 --- /dev/null +++ b/scripts/tasks/features/attestations.ts @@ -0,0 +1,251 @@ +// spell-checker:ignore tlog +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { Bundle, BundleVerifier } from "sigstore"; +import { createVerifier } from "sigstore"; +import { z } from "zod"; + +import { githubApiHeaders } from "./github.js"; +import { uncompress } from "./snappy.js"; + +const releasePredicateType = "https://in-toto.io/attestation/release/v0.2"; +const slsaPredicateType = "https://slsa.dev/provenance/v1"; +const releaseIdentity = "https://dotcom.releases.github.com"; +const githubActionsIssuer = "https://token.actions.githubusercontent.com"; +const releaseMirrorURL = "https://tuf-repo.github.com"; +const slsaMirrorURL = "https://tuf-repo-cdn.sigstore.dev"; + +const bundleSchema = z.looseObject({ + mediaType: z.string(), + dsseEnvelope: z.looseObject({ payload: z.string(), payloadType: z.string() }), +}); + +const attestationEntrySchema = z.looseObject({ bundle_url: z.string() }); + +const attestationsSchema = z.object({ attestations: z.array(attestationEntrySchema) }); + +const statementTypeSchema = z.looseObject({ predicateType: z.string() }); + +const releaseStatementSchema = z.looseObject({ + predicateType: z.literal(releasePredicateType), + subject: z.array( + z.looseObject({ name: z.string().optional(), digest: z.looseObject({ sha256: z.string().optional() }) }), + ), + predicate: z.looseObject({ repository: z.string(), tag: z.string(), purl: z.string() }), +}); + +const slsaStatementSchema = z.looseObject({ + predicateType: z.literal(slsaPredicateType), + subject: z.array(z.looseObject({ digest: z.looseObject({ sha256: z.string().optional() }) })), +}); + +type AttestationEntry = z.infer; +type ReleaseStatement = z.infer; + +interface Policy { + mirrorURL: string; + identity: string; + issuer?: string; + tlogThreshold?: number; + ctLogThreshold?: number; +} + +const releasePolicy: Policy = { + mirrorURL: releaseMirrorURL, + identity: releaseIdentity, + tlogThreshold: 0, + ctLogThreshold: 0, +}; + +const slsaPolicy = (identity: string): Policy => ({ + mirrorURL: slsaMirrorURL, + identity, + issuer: githubActionsIssuer, +}); + +export interface AttestationTarget { + repository: string; + sha256: string; + url: string; + tag: string; + purl: string; + identity: string; +} + +export interface Attestations { + verify(target: AttestationTarget): Promise; + [Symbol.asyncDispose](): Promise; +} + +const escapeRegExp = (text: string): string => text.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + +const describe = (target: AttestationTarget, predicateType: string): string => + `${target.repository} sha256:${target.sha256} ${predicateType}`; + +const messageOf = (error: unknown): string => (error instanceof Error ? error.message : String(error)); + +const assetName = (url: string): string => { + const name = new URL(url).pathname.split("/").at(-1); + if (name === undefined || name === "") { + throw new Error(`${url}: the URL carries no file name`); + } + return decodeURIComponent(name); +}; + +const statementOf = (bundle: Bundle, source: string): unknown => { + const envelope = bundle.dsseEnvelope; + if (envelope === undefined) { + throw new Error(`${source}: the bundle has no DSSE envelope`); + } + try { + return JSON.parse(Buffer.from(envelope.payload, "base64").toString("utf8")) as unknown; + } catch (error) { + throw new Error(`${source}: the DSSE payload is not JSON: ${messageOf(error)}`, { cause: error }); + } +}; + +const bundleOf = async (entry: AttestationEntry, source: string, index: number): Promise => { + const url = entry.bundle_url; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`${source}: attestation ${index}: GET ${url} → HTTP ${response.status}`); + } + try { + const json: unknown = JSON.parse( + Buffer.from(uncompress(new Uint8Array(await response.arrayBuffer()))).toString("utf8"), + ); + return bundleSchema.parse(json) as Bundle; + } catch (error) { + throw new Error(`${source}: attestation ${index}: cannot decode the bundle at ${url}: ${messageOf(error)}`, { + cause: error, + }); + } +}; + +const assertReleaseClaims = (statement: ReleaseStatement, target: AttestationTarget, source: string): void => { + const { predicate } = statement; + if (predicate.repository !== target.repository) { + throw new Error(`${source}: release repository ${predicate.repository} does not match ${target.repository}`); + } + if (predicate.tag !== target.tag) { + throw new Error(`${source}: release tag ${predicate.tag} does not match ${target.tag}`); + } + if (predicate.purl !== target.purl) { + throw new Error(`${source}: release purl ${predicate.purl} does not match ${target.purl}`); + } + const name = assetName(target.url); + if (!statement.subject.some((subject) => subject.name === name && subject.digest.sha256 === target.sha256)) { + throw new Error(`${source}: no subject names ${name} with sha256 ${target.sha256}`); + } +}; + +export const createAttestations = async (): Promise => { + const cachePath = await mkdtemp(path.join(tmpdir(), "devcontainer-attestations-")); + const verifiers = new Map>(); + + const anchorOf = async (policy: Policy, key: string): Promise => { + const url = `${policy.mirrorURL}/1.root.json`; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`GET ${url} → HTTP ${response.status}`); + } + const anchorPath = path.join(cachePath, "anchors", `${createHash("sha256").update(key).digest("hex")}.json`); + await mkdir(path.dirname(anchorPath), { recursive: true }); + await writeFile(anchorPath, new Uint8Array(await response.arrayBuffer())); + return anchorPath; + }; + + const verifierFor = (policy: Policy): Promise => { + const key = JSON.stringify([ + policy.mirrorURL, + policy.identity, + policy.issuer, + policy.tlogThreshold, + policy.ctLogThreshold, + ]); + let verifier = verifiers.get(key); + if (verifier === undefined) { + verifier = (async () => { + const anchorPath = await anchorOf(policy, key); + return createVerifier({ + tufMirrorURL: policy.mirrorURL, + tufRootPath: anchorPath, + tufCachePath: cachePath, + certificateIdentityURI: `^${escapeRegExp(policy.identity)}$`, + ...(policy.issuer === undefined ? {} : { certificateIssuer: policy.issuer }), + ...(policy.tlogThreshold === undefined ? {} : { tlogThreshold: policy.tlogThreshold }), + ...(policy.ctLogThreshold === undefined ? {} : { ctLogThreshold: policy.ctLogThreshold }), + }); + })(); + verifiers.set(key, verifier); + } + return verifier; + }; + + const verify = async (target: AttestationTarget): Promise => { + const url = `https://api.github.com/repos/${target.repository}/attestations/sha256:${target.sha256}`; + const response = await fetch(url, { headers: githubApiHeaders() }); + if (!response.ok) { + throw new Error(`GET ${url} → HTTP ${response.status}`); + } + const parsed = attestationsSchema.safeParse(await response.json()); + if (!parsed.success) { + throw new Error(`GET ${url}: ${z.prettifyError(parsed.error)}`); + } + + let releaseStatements = 0; + let slsaStatements = 0; + for (const [index, entry] of parsed.data.attestations.entries()) { + const bundle = await bundleOf(entry, url, index); + const type = statementTypeSchema.safeParse(statementOf(bundle, `${url}: attestation ${index}`)); + if (!type.success) { + throw new Error(`${url}: attestation ${index} has no predicateType`); + } + if (type.data.predicateType === releasePredicateType) { + const source = `${describe(target, releasePredicateType)}: attestation ${index}`; + const statement = releaseStatementSchema.safeParse(statementOf(bundle, source)); + if (!statement.success) { + throw new Error(`${source}: ${z.prettifyError(statement.error)}`); + } + try { + await Promise.resolve((await verifierFor(releasePolicy)).verify(bundle)); + } catch (error) { + throw new Error(`${source}: verification failed: ${messageOf(error)}`, { cause: error }); + } + assertReleaseClaims(statement.data, target, source); + releaseStatements++; + } else if (type.data.predicateType === slsaPredicateType) { + const source = `${describe(target, slsaPredicateType)}: attestation ${index}`; + const statement = slsaStatementSchema.safeParse(statementOf(bundle, source)); + if (!statement.success) { + throw new Error(`${source}: ${z.prettifyError(statement.error)}`); + } + try { + await Promise.resolve((await verifierFor(slsaPolicy(target.identity))).verify(bundle)); + } catch (error) { + throw new Error(`${source}: verification failed: ${messageOf(error)}`, { cause: error }); + } + if (!statement.data.subject.some((subject) => subject.digest.sha256 === target.sha256)) { + throw new Error(`${source}: no subject carries sha256 ${target.sha256}`); + } + slsaStatements++; + } + } + if (releaseStatements === 0) { + throw new Error(`${url}: no release attestation`); + } + if (slsaStatements === 0) { + throw new Error(`${url}: no SLSA v1 provenance attestation`); + } + }; + + return { + verify, + async [Symbol.asyncDispose]() { + await rm(cachePath, { recursive: true, force: true }); + }, + }; +}; diff --git a/scripts/tasks/features/config.ts b/scripts/tasks/features/config.ts index 53b1d10..1f655fe 100644 --- a/scripts/tasks/features/config.ts +++ b/scripts/tasks/features/config.ts @@ -110,9 +110,7 @@ export const formatConfig = async (document: unknown, configPath: string): Promi export interface ConfigFile { parse: (text: string, source: string) => T; read: () => Promise; - /** The document's revision on disk, 1 when the file does not exist yet. */ revision: () => Promise; - /** Validates the document, then writes it canonically. */ write: (config: T) => Promise; } diff --git a/scripts/tasks/features/features.ts b/scripts/tasks/features/features.ts index ac2ba9f..75ac5db 100644 --- a/scripts/tasks/features/features.ts +++ b/scripts/tasks/features/features.ts @@ -1,12 +1,14 @@ import { codeServer } from "./code-server/feature.js"; import { dotnet } from "./dotnet/feature.js"; import type { Feature } from "./feature.js"; +import { githubCli } from "./github-cli/feature.js"; import { ripgrep } from "./ripgrep/feature.js"; import { uv } from "./uv/feature.js"; -export const features: Readonly> = { +export const features: Readonly> = { "code-server": codeServer, dotnet, + "github-cli": githubCli, ripgrep, uv, }; diff --git a/scripts/tasks/features/github-cli/feature.ts b/scripts/tasks/features/github-cli/feature.ts new file mode 100644 index 0000000..4d06360 --- /dev/null +++ b/scripts/tasks/features/github-cli/feature.ts @@ -0,0 +1,31 @@ +import type { ConfigDocument } from "@/scripts/tasks/features/config.js"; +import type { Feature, FeatureStage } from "@/scripts/tasks/features/feature.js"; + +import type { FeatureConfig } from "./generateConfig.js"; +import { configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; +import { derive, ensurePayload, loadSection, payloadPresent } from "./payload.js"; + +const component = "github-cli"; + +export const documentOf = (config: FeatureConfig): ConfigDocument => ({ + revision: config.revision, + namespaces: [{ name: component, versions: config.versions, aliases: config.aliases }], +}); + +const stage: FeatureStage = { subdir: [], files: ["install"], binary: false }; + +export const githubCli: Feature = { + id: "github-cli", + configPath, + components: [component], + document: async () => documentOf(await readConfig()), + parseDocument: (text, source) => documentOf(parseConfig(text, source)), + generateConfig, + repository: () => "github-cli", + stage: () => stage, + payload: { + ensure: async (_component, key, arch, force) => ensurePayload(await loadSection(key), arch, force), + present: async (_component, key, arch) => payloadPresent(derive(await loadSection(key), arch)), + }, + gitTag: (_component, canonical) => `features-github-cli-${canonical}`, +}; diff --git a/scripts/tasks/features/github-cli/generateConfig.ts b/scripts/tasks/features/github-cli/generateConfig.ts new file mode 100644 index 0000000..2b2ae2c --- /dev/null +++ b/scripts/tasks/features/github-cli/generateConfig.ts @@ -0,0 +1,78 @@ +import path from "node:path"; + +import { z } from "zod"; + +import { projectRoot } from "@/scripts/project.js"; +import { checkTagNamespace, configFile } from "@/scripts/tasks/features/config.js"; + +import type { DockerArch } from "../architecture.js"; +import { dockerArchOptions, ridFor } from "../architecture.js"; +import type { Attestations } from "../attestations.js"; +import { createAttestations } from "../attestations.js"; +import type { TrackedRelease } from "../githubReleases.js"; +import { artifactOf, trackedRelease, versionAliases } from "../githubReleases.js"; + +const repository = "cli/cli"; +const identity = "https://github.com/cli/cli/.github/workflows/deployment.yml@refs/heads/trunk"; + +export const configPath = path.resolve(projectRoot, "features/github-cli/config.json"); + +const artifactSchema = z.strictObject({ url: z.string(), sha256: z.string() }); + +const versionEntrySchema = z.strictObject({ + "linux-x64": artifactSchema, + "linux-arm64": artifactSchema, +}); + +const aliasTableSchema = z.record(z.string(), z.array(z.string())); + +export const featureConfigSchema = z.strictObject({ + revision: z.int().min(1), + versions: z.record(z.string(), versionEntrySchema), + aliases: aliasTableSchema, +}); + +export type Artifact = z.infer; +export type VersionEntry = z.infer; +export type FeatureConfig = z.infer; + +const validateConfig = (config: FeatureConfig, source: string): void => { + checkTagNamespace({ + source, + revision: config.revision, + versions: config.versions, + aliases: config.aliases, + requiredClaims: ["latest"], + }); +}; + +const config = configFile({ path: configPath, schema: featureConfigSchema, validate: validateConfig }); + +export const parseConfig = config.parse; +export const readConfig = config.read; + +const assetName = (version: string, arch: DockerArch): string => `gh_${version}_linux_${arch}.tar.gz`; + +const resolveArtifacts = async (attestations: Attestations, release: TrackedRelease): Promise => { + const tag = `v${release.version}`; + const purl = `pkg:github/${repository}@${tag}`; + const entries = await Promise.all( + dockerArchOptions.map(async (arch) => { + const artifact = artifactOf(release, assetName(release.version, arch)); + await attestations.verify({ repository, sha256: artifact.sha256, url: artifact.url, tag, purl, identity }); + return [ridFor(arch), artifact] as const; + }), + ); + return Object.fromEntries(entries) as VersionEntry; +}; + +export const generateConfig = async (): Promise => { + await using attestations = await createAttestations(); + const release = await trackedRelease(repository); + const document: FeatureConfig = { + revision: await config.revision(), + versions: { [release.version]: await resolveArtifacts(attestations, release) }, + aliases: { [release.version]: versionAliases(release) }, + }; + await config.write(document); +}; diff --git a/scripts/tasks/features/github-cli/payload.ts b/scripts/tasks/features/github-cli/payload.ts new file mode 100644 index 0000000..aa81e48 --- /dev/null +++ b/scripts/tasks/features/github-cli/payload.ts @@ -0,0 +1,51 @@ +import path from "node:path"; + +import type { DockerArch } from "../architecture.js"; +import { isFile, ridFor } from "../architecture.js"; +import { payloadPath } from "../paths.js"; +import type { Tarball } from "../tarball.js"; +import { stageTarball } from "../tarball.js"; +import type { FeatureConfig, VersionEntry } from "./generateConfig.js"; +import { configPath, readConfig } from "./generateConfig.js"; + +export interface PayloadSection { + key: string; + entry: VersionEntry; +} + +export interface Payload { + cache: string; + present: string; + tarball: Tarball; +} + +export const sectionOf = (config: FeatureConfig, key: string): PayloadSection => { + const entry = config.versions[key]; + if (entry === undefined) { + throw new Error(`Version ${key} is not in ${configPath}`); + } + return { key, entry }; +}; + +export const loadSection = async (key: string): Promise => sectionOf(await readConfig(), key); + +export const derive = (section: PayloadSection, arch: DockerArch): Payload => { + const cache = payloadPath("github-cli", "github-cli", section.key, arch); + const artifact = section.entry[ridFor(arch)]; + return { + cache, + present: path.join(cache, "bin/gh"), + tarball: { cache, url: artifact.url, sha256: artifact.sha256 }, + }; +}; + +export const payloadPresent = (payload: Payload): Promise => isFile(payload.present); + +export const ensurePayload = async (section: PayloadSection, arch: DockerArch, force: boolean): Promise => { + const payload = derive(section, arch); + if (!force && (await payloadPresent(payload))) { + console.log(`Payload already present at ${payload.cache}; skipping fetch`); + return; + } + await stageTarball(payload.tarball); +}; diff --git a/scripts/tasks/features/github.ts b/scripts/tasks/features/github.ts new file mode 100644 index 0000000..5858b93 --- /dev/null +++ b/scripts/tasks/features/github.ts @@ -0,0 +1,7 @@ +export const githubApiHeaders = (): Readonly> => { + const token = process.env.GH_TOKEN; + return { + accept: "application/vnd.github+json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }; +}; diff --git a/scripts/tasks/features/githubReleases.ts b/scripts/tasks/features/githubReleases.ts index f395ae2..b52807a 100644 --- a/scripts/tasks/features/githubReleases.ts +++ b/scripts/tasks/features/githubReleases.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { githubApiHeaders } from "./github.js"; + export type Version = readonly [number, number, number]; export interface ReleaseAsset { @@ -9,7 +11,7 @@ export interface ReleaseAsset { } export interface TrackedRelease { - /** The release tag with its `v` prefix removed. */ + // The release tag with its `v` prefix removed. version: string; number: Version; assets: readonly ReleaseAsset[]; @@ -55,7 +57,7 @@ const fetchReleases = async (repository: string): Promise = []; for (let page = 1; ; page++) { const url = `https://api.github.com/repos/${repository}/releases?per_page=100&page=${page}`; - const response = await fetch(url, { headers: { accept: "application/vnd.github+json" } }); + const response = await fetch(url, { headers: githubApiHeaders() }); if (!response.ok) { throw new Error(`GET ${url} → HTTP ${response.status}`); } @@ -70,7 +72,7 @@ const fetchReleases = async (repository: string): Promise => { const releases = await fetchReleases(repository); const candidates = releases.flatMap((release): TrackedRelease[] => { @@ -99,7 +101,7 @@ export const trackedRelease = async (repository: string, minimum?: Version): Pro return candidates.reduce((best, candidate) => (compare(candidate.number, best.number) > 0 ? candidate : best)); }; -/** The url and sha256 digest of the one release asset named `name`. */ +// The url and sha256 digest of the one release asset named `name`. export const artifactOf = (release: TrackedRelease, name: string): { url: string; sha256: string } => { const matches = release.assets.filter((asset) => asset.name === name); if (matches.length !== 1) { @@ -113,7 +115,6 @@ export const artifactOf = (release: TrackedRelease, name: string): { url: string return { url: asset.url, sha256: match[1] }; }; -/** The `.`, `` and `latest` names a release claims. */ export const versionAliases = (release: TrackedRelease): string[] => { const [major, minor] = release.number; return [`${major}.${minor}`, `${major}`, "latest"]; diff --git a/scripts/tasks/features/snappy.ts b/scripts/tasks/features/snappy.ts new file mode 100644 index 0000000..2d7a2a9 --- /dev/null +++ b/scripts/tasks/features/snappy.ts @@ -0,0 +1,83 @@ +// Block format: https://github.com/google/snappy/blob/main/format_description.txt + +const readLittleEndian = (input: Uint8Array, offset: number, width: number): number => { + if (offset + width > input.length) { + throw new Error("truncated input"); + } + let value = 0; + for (let index = 0; index < width; index++) { + value += input[offset + index] * 2 ** (8 * index); + } + return value; +}; + +export const uncompress = (input: Uint8Array): Uint8Array => { + let cursor = 0; + const nextByte = (): number => { + if (cursor === input.length) { + throw new Error("truncated input"); + } + return input[cursor++]; + }; + + let length = 0; + for (let width = 0; ; width++) { + const value = nextByte(); + if (width === 4 && value >= 16) { + throw new Error("the uncompressed length is not a 32-bit varint"); + } + length += (value & 0x7f) * 2 ** (7 * width); + if (value < 0x80) { + break; + } + } + + const output = new Uint8Array(length); + let written = 0; + while (cursor < input.length) { + const tag = nextByte(); + const type = tag & 0x03; + if (type === 0) { + const code = tag >>> 2; + // Codes 60..63 encode the literal length in 1..4 little-endian bytes. + const literalLength = code < 60 ? code + 1 : readLittleEndian(input, cursor, code - 59) + 1; + if (code >= 60) { + cursor += code - 59; + } + if (cursor + literalLength > input.length) { + throw new Error("truncated literal"); + } + if (output.length - written < literalLength) { + throw new Error("the literal exceeds the declared length"); + } + output.set(input.subarray(cursor, cursor + literalLength), written); + cursor += literalLength; + written += literalLength; + } else { + const copyLength = type === 1 ? ((tag >>> 2) & 0x07) + 4 : (tag >>> 2) + 1; + let offset: number; + if (type === 1) { + offset = ((tag >>> 5) << 8) | nextByte(); + } else { + const width = type === 2 ? 2 : 4; + offset = readLittleEndian(input, cursor, width); + cursor += width; + } + if (offset === 0 || offset > written) { + throw new Error("the copy offset points outside the decoded data"); + } + if (output.length - written < copyLength) { + throw new Error("the copy exceeds the declared length"); + } + // A copy may overlap the bytes it produces, so it is written one byte at a time. + for (let index = 0; index < copyLength; index++) { + output[written + index] = output[written - offset + index]; + } + written += copyLength; + } + } + if (written !== output.length) { + throw new Error(`the stream ended after ${written} of ${output.length} bytes`); + } + return output; +}; diff --git a/scripts/tasks/features/tarball.ts b/scripts/tasks/features/tarball.ts index e6c3b52..bfde5c5 100644 --- a/scripts/tasks/features/tarball.ts +++ b/scripts/tasks/features/tarball.ts @@ -7,15 +7,12 @@ import { isFile } from "./architecture.js"; import { download } from "./download.js"; export interface Tarball { - /** The payload directory the extracted archive replaces in one rename. */ cache: string; url: string; sha256: string; - /** The members to extract, under the archive's single top directory; every member when absent. */ extract?: { top: string; entries: readonly string[] }; } -/** Fetches and extracts `tarball`; its cache is replaced only once the archive has passed every check. */ export const stageTarball = async (tarball: Tarball): Promise => { const tempPath = `${tarball.cache}.tmp`; await rm(tempPath, { recursive: true, force: true }); diff --git a/scripts/tasks/features/uv/attestations.ts b/scripts/tasks/features/uv/attestations.ts deleted file mode 100644 index 50b6103..0000000 --- a/scripts/tasks/features/uv/attestations.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { Bundle, BundleVerifier } from "sigstore"; -import { createVerifier } from "sigstore"; -import { z } from "zod"; - -const predicateTypeSlsaV1 = "https://slsa.dev/provenance/v1"; -const certificateIssuer = "https://token.actions.githubusercontent.com"; - -const statementSchema = z.object({ - predicateType: z.string(), - subject: z.array(z.object({ digest: z.object({ sha256: z.string() }) })), -}); - -const attestationsSchema = z.object({ - attestations: z.array( - z.object({ - bundle: z.looseObject({ - mediaType: z.string(), - dsseEnvelope: z.looseObject({ payload: z.string(), payloadType: z.string() }), - }), - }), - ), -}); - -const verifiers = new Map>(); - -const escapeRegExp = (text: string): string => text.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); - -const verifierFor = (slug: string): Promise => { - let verifier = verifiers.get(slug); - if (verifier === undefined) { - const identity = escapeRegExp(`https://github.com/${slug}/.github/workflows/release.yml@refs/heads/main`); - verifier = createVerifier({ certificateIssuer, certificateIdentityURI: `^${identity}$` }); - verifiers.set(slug, verifier); - } - return verifier; -}; - -const decodeStatement = (envelope: { payload: string }): unknown => - JSON.parse(Buffer.from(envelope.payload, "base64").toString("utf8")); - -export const verifyAttestation = async (slug: string, sha256: string): Promise => { - const url = `https://api.github.com/repos/${slug}/attestations/sha256:${sha256}`; - const response = await fetch(url, { headers: { accept: "application/vnd.github+json" } }); - if (!response.ok) { - throw new Error(`GET ${url} → HTTP ${response.status}`); - } - const result = attestationsSchema.safeParse(await response.json()); - if (!result.success) { - throw new Error(`GET ${url}: ${z.prettifyError(result.error)}`); - } - const provenance = result.data.attestations.flatMap(({ bundle }) => { - const statement = statementSchema.safeParse(decodeStatement(bundle.dsseEnvelope)); - return statement.success && statement.data.predicateType === predicateTypeSlsaV1 - ? [{ bundle, statement: statement.data }] - : []; - }); - if (provenance.length === 0) { - throw new Error(`${url}: no SLSA v1 provenance attestation`); - } - const verifier = await verifierFor(slug); - for (const { bundle, statement } of provenance) { - // The API returns the serialized bundle exactly as sigstore's verifier takes it; the verifier fully validates - // the shape when it parses the bundle. - verifier.verify(bundle as Bundle); - if (!statement.subject.some((subject) => subject.digest.sha256 === sha256)) { - throw new Error(`${url}: no statement subject carries sha256 ${sha256}`); - } - } -}; diff --git a/scripts/tasks/features/uv/generateConfig.ts b/scripts/tasks/features/uv/generateConfig.ts index 5f16a30..bc3ba33 100644 --- a/scripts/tasks/features/uv/generateConfig.ts +++ b/scripts/tasks/features/uv/generateConfig.ts @@ -7,13 +7,17 @@ import { checkTagNamespace, configFile } from "@/scripts/tasks/features/config.j import type { DockerArch, Rid } from "../architecture.js"; import { dockerArchOptions, ridFor, rids } from "../architecture.js"; -import { verifyAttestation } from "./attestations.js"; +import type { Attestations } from "../attestations.js"; +import { createAttestations } from "../attestations.js"; import { gnuTriples, muslTriples } from "./paths.js"; const uvIndexUrl = "https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson"; const pythonIndexUrl = "https://raw.githubusercontent.com/astral-sh/versions/main/v1/python-build-standalone.ndjson"; const uvSlug = "astral-sh/uv"; const pythonSlug = "astral-sh/python-build-standalone"; +const uvIdentity = "https://github.com/astral-sh/uv/.github/workflows/release.yml@refs/heads/main"; +const pythonIdentity = + "https://github.com/astral-sh/python-build-standalone/.github/workflows/release.yml@refs/heads/main"; const stablePattern = /^\d+\.\d+\.\d+$/u; const buildPattern = /^\d+$/u; @@ -74,6 +78,20 @@ export const featureConfigSchema = z.strictObject({ type IndexArtifact = z.infer; type IndexRelease = z.infer; +interface ReleaseIdentity { + repository: string; + tag: string; + purl: string; + identity: string; +} + +const releaseOf = (repository: string, tag: string, identity: string): ReleaseIdentity => ({ + repository, + tag, + purl: `pkg:github/${repository}@${tag}`, + identity, +}); + export type Artifact = z.infer; export type UvVersion = z.infer; export type PythonVersion = z.infer; @@ -100,7 +118,8 @@ const parseIndex = (text: string, source: string): IndexRelease[] => }); const resolveArtifacts = async ( - slug: string, + attestations: Attestations, + release: ReleaseIdentity, version: string, artifacts: readonly IndexArtifact[], platforms: Record, @@ -121,7 +140,8 @@ const resolveArtifacts = async ( }), ) as Record; for (const rid of rids) { - await verifyAttestation(slug, resolved[rid].sha256); + const artifact = resolved[rid]; + await attestations.verify({ ...release, sha256: artifact.sha256, url: artifact.url }); } return resolved; }; @@ -176,7 +196,10 @@ const trackedPythonReleases = (releases: readonly IndexRelease[]): { build: stri return { build, entries }; }; -const pythonVersions = async (releases: readonly IndexRelease[]): Promise => { +const pythonVersions = async ( + attestations: Attestations, + releases: readonly IndexRelease[], +): Promise => { const { build, entries } = trackedPythonReleases(releases); const resolved = await Promise.all( entries.map(async (release) => { @@ -186,7 +209,14 @@ const pythonVersions = async (releases: readonly IndexRelease[]): Promise => { + await using attestations = await createAttestations(); const uvRelease = trackedUvRelease(parseIndex(await fetchText(uvIndexUrl), uvIndexUrl)); const pythonReleases = parseIndex(await fetchText(pythonIndexUrl), pythonIndexUrl); const [major, minor] = uvRelease.version.split("."); @@ -240,7 +271,8 @@ export const generateConfig = async (): Promise => { uv: { versions: { [uvRelease.version]: await resolveArtifacts( - uvSlug, + attestations, + releaseOf(uvSlug, uvRelease.version, uvIdentity), uvRelease.version, uvRelease.artifacts, muslTriples, @@ -249,7 +281,7 @@ export const generateConfig = async (): Promise => { }, aliases: { [uvRelease.version]: [`${major}.${minor}`, "latest"] }, }, - python: await pythonVersions(pythonReleases), + python: await pythonVersions(attestations, pythonReleases), }, }; await config.write(document); diff --git a/scripts/verify/github-cli/selection.ts b/scripts/verify/github-cli/selection.ts new file mode 100644 index 0000000..296b609 --- /dev/null +++ b/scripts/verify/github-cli/selection.ts @@ -0,0 +1,24 @@ +import { claimedKey } from "@/scripts/tasks/features/config.js"; +import { documentOf, githubCli as githubCliFeature } from "@/scripts/tasks/features/github-cli/feature.js"; +import type { FeatureConfig } from "@/scripts/tasks/features/github-cli/generateConfig.js"; +import { configPath, readConfig } from "@/scripts/tasks/features/github-cli/generateConfig.js"; +import { imageEntry } from "@/scripts/tasks/features/imageEntries.js"; + +import type { ComponentImage } from "../lifecycle.js"; + +export type Selection = ComponentImage; + +const selectedKey = (config: FeatureConfig, key: string | undefined): string => + key ?? claimedKey(config.aliases, "latest", configPath); + +const derive = async (channel: string, prefix: string, key?: string): Promise => { + const config = await readConfig(); + const selected = selectedKey(config, key); + const { canonical } = imageEntry(githubCliFeature, documentOf(config), prefix, channel, { + component: "github-cli", + key: selected, + }); + return { component: "github-cli", key: selected, ref: canonical }; +}; + +export const selection = { derive } as const; diff --git a/scripts/verify/github-cli/test-image.Dockerfile b/scripts/verify/github-cli/test-image.Dockerfile new file mode 100644 index 0000000..12b51a3 --- /dev/null +++ b/scripts/verify/github-cli/test-image.Dockerfile @@ -0,0 +1,7 @@ +# check=skip=InvalidDefaultArgInFrom +ARG GITHUB_CLI_IMAGE_REF + +FROM ${GITHUB_CLI_IMAGE_REF} AS github-cli + +FROM mcr.microsoft.com/devcontainers/base:debian +COPY --from=github-cli /features/github-cli/ /opt/devcontainer-config/features/github-cli/ diff --git a/scripts/verify/github-cli/verify.ts b/scripts/verify/github-cli/verify.ts new file mode 100644 index 0000000..aa16193 --- /dev/null +++ b/scripts/verify/github-cli/verify.ts @@ -0,0 +1,142 @@ +// spell-checker:ignore compinit +import path from "node:path"; + +import { z } from "zod"; + +import { parseVerifyArguments, selectionSchema, targetSchema } from "../arguments.js"; +import { lifecycle } from "../lifecycle.js"; +import type { Exec } from "../run.js"; +import { runSuite } from "../run.js"; +import { userInit } from "../userInit.js"; +import { selection } from "./selection.js"; + +const names = ["default", "smoke"] as const; +type ScenarioName = (typeof names)[number]; + +const scenarioSelectionSchema = selectionSchema(z.enum(names), names, ["default"]); + +const targetSchemaForFeature = targetSchema(z.literal("github-cli")); + +const testImagePrefix = "features-verify-github-cli"; +const testImageDockerfile = path.join(import.meta.dirname, "test-image.Dockerfile"); +const featureDir = "/opt/devcontainer-config/features/github-cli"; + +const completionFiles = [ + "/usr/local/share/bash-completion/completions/gh", + "/usr/local/share/zsh/site-functions/_gh", + "/usr/local/share/fish/vendor_completions.d/gh.fish", +]; + +const bashCompletionRegistration: Exec = { + command: [ + "bash", + "-lc", + "source /usr/share/bash-completion/bash_completion; _completion_loader gh; complete -p gh >/dev/null && echo registered", + ], + expect: { stdout: ["registered"] }, +}; + +const zshCompletionRegistration: Exec = { + command: ["zsh", "-f", "-c", "autoload -Uz compinit; compinit -D; print -r -- ${_comps[gh]:-NONE}"], + expect: { stdout: ["_gh"] }, +}; + +const defaultExecs = (version: string, residuePaths: readonly string[]): Exec[] => [ + { + command: ["sh", "-c", "gh --version | awk 'NR == 1 { print $1, $2, $3 }'"], + expect: { stdout: [`gh version ${version}`] }, + }, + { command: ["gh", "--help"], expect: { exitCode: 0 } }, + { + command: ["sh", "-c", "command -v gh"], + expect: { stdout: ["/usr/local/bin/gh"] }, + }, + { + command: ["readlink", "-f", "/usr/local/bin/gh"], + expect: { stdout: [`${featureDir}/bin/gh`] }, + }, + { + command: [ + "sh", + "-c", + `diff -r ${featureDir}/share/man/man1 /usr/local/share/man/man1 && cmp ${featureDir}/LICENSE /usr/local/share/doc/gh/LICENSE && echo copied`, + ], + expect: { stdout: ["copied"] }, + }, + { + command: ["sh", "-c", `for file in ${completionFiles.join(" ")}; do test -s "$file" || exit 1; done; echo present`], + expect: { stdout: ["present"] }, + }, + bashCompletionRegistration, + zshCompletionRegistration, + { + command: ["sh", "-c", "grep -qxF 'export GH_TELEMETRY=false' /etc/profile.d/gh.sh"], + expect: { stdout: [] }, + }, + { + command: ["bash", "-lc", 'printf "%s\\n" "${GH_TELEMETRY:-NONE}"'], + expect: { stdout: ["false"] }, + }, + { + command: [ + "sh", + "-c", + `for target in ${residuePaths.join(" ")}; do test ! -e "$target" || { echo "residue $target"; exit 1; }; done; echo clean`, + ], + expect: { stdout: ["clean"] }, + }, +]; + +const smokeExecs = (version: string): Exec[] => [ + { + command: ["sh", "-c", "gh --version | awk 'NR == 1 { print $1, $2, $3 }'"], + expect: { stdout: [`gh version ${version}`] }, + }, + { + command: ["sh", "-c", "command -v gh"], + expect: { stdout: ["/usr/local/bin/gh"] }, + }, + { command: ["gh", "--help"], expect: { exitCode: 0 } }, +]; + +const execsOf: Record readonly Exec[]> = { + default: defaultExecs, + smoke: smokeExecs, +}; + +export const verify = async (argv: string[]): Promise => { + const { options, scenarios: requested, component, version } = await parseVerifyArguments(argv); + const selected = scenarioSelectionSchema.parse(requested); + const target = targetSchemaForFeature.parse({ component, version }); + const derived = await selection.derive(options.channel, options.prefix, target?.key); + console.log(`>>> verify github-cli: selection version=${derived.key} ref=${derived.ref}`); + + const residuePaths = [ + "/root/.config/gh", + "/root/.local/state/gh", + "/root/.local/share/gh", + "/root/.cache/gh", + ...(await Promise.all( + ["XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"].map(async (variable) => + path.posix.join(await userInit.xdgPath(variable), "gh"), + ), + )), + ]; + + await runSuite({ + ...options, + name: "github-cli", + create: (root) => + selected.map((name) => ({ + name, + workspace: lifecycle.workspaceAt(root, name), + testImage: { + tag: `${testImagePrefix}:${derived.key}`, + components: [derived], + dockerfile: testImageDockerfile, + args: [`GITHUB_CLI_IMAGE_REF=${derived.ref}`], + }, + execs: execsOf[name](derived.key, residuePaths), + })), + }); +};