From 5bf97c6416bb2bf2563dda4cd228a6a63efeecbc Mon Sep 17 00:00:00 2001 From: lcawl Date: Tue, 8 Sep 2026 12:08:52 -0500 Subject: [PATCH] Add changelog unpack command --- docs/cli-schema.json | 108 ++++++ docs/cli/changelog/cmd-bundle.md | 2 +- docs/cli/changelog/cmd-unpack.md | 36 ++ docs/cli/changelog/cmd-upload.md | 4 +- docs/cli/changelog/index.md | 2 +- docs/data/release-notes/create.md | 10 +- .../ReleaseNotes/ChangelogTextUtilities.cs | 29 ++ .../Creation/ChangelogCreationService.cs | 64 +++- .../Creation/ChangelogFileWriter.cs | 53 +-- .../Creation/ChangelogUnpackService.cs | 269 ++++++++++++++ .../docs-builder/Commands/ChangelogCommand.cs | 46 +++ .../Changelogs/Create/ChangelogUnpackTests.cs | 349 ++++++++++++++++++ .../ChangelogTextUtilitiesTests.cs | 17 + 13 files changed, 957 insertions(+), 32 deletions(-) create mode 100644 docs/cli/changelog/cmd-unpack.md create mode 100644 src/services/Elastic.Changelog/Creation/ChangelogUnpackService.cs create mode 100644 tests/Elastic.Changelog.Tests/Changelogs/Create/ChangelogUnpackTests.cs diff --git a/docs/cli-schema.json b/docs/cli-schema.json index b377e7c095..91830200c1 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -4918,6 +4918,114 @@ } ] }, + { + "path": [ + "changelog" + ], + "name": "unpack", + "summary": "Recreate individual changelog YAML files from a bundle using changelog add and changelog note.", + "notes": "Writes one file per bundle entry through the same writers as changelog add (PR-anchored)\nand changelog note (no PR, or products with versions). Filenames follow add and note rules,\nnot the bundle file.name provenance. Checksums are not reproduced.\nBundles that were scrubbed during changelog upload (public-bucket copies) have their\nprivate PRs and issues removed, so they are less likely to unpack successfully. Prefer the\nprivate-side bundle YAML.", + "usage": "docs-builder changelog unpack \u003Cbundle\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "positional", + "name": "bundle", + "type": "string", + "required": true, + "summary": "Local bundle or amend YAML file to unpack. The file must exist on disk; CDN locators such as /bundle/{product}/{file}.yaml are not accepted. A parent bundle is merged with sibling .amend-* bundles first (same as changelog render). An amend bundle unpacks only that file\u0027s entries; exclude-entries are skipped.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "config", + "type": "string", + "required": false, + "summary": "Path to changelog.yml. Defaults to docs/changelog.yml. Type and area validation uses this configuration.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "output", + "type": "string", + "required": false, + "summary": "Directory for written changelog files. Defaults to bundle.directory in changelog.yml, then the current directory." + }, + { + "role": "flag", + "name": "concise", + "type": "boolean", + "required": false, + "summary": "Omit schema reference comments from generated YAML, matching changelog add --concise.", + "defaultValue": "false" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ] + }, { "path": [ "changelog" diff --git a/docs/cli/changelog/cmd-bundle.md b/docs/cli/changelog/cmd-bundle.md index ed4ac7857b..b8f424296f 100644 --- a/docs/cli/changelog/cmd-bundle.md +++ b/docs/cli/changelog/cmd-bundle.md @@ -119,7 +119,7 @@ docs-builder changelog bundle serverless-release 2026-08-13 \ ## Bundles are self-contained -Every bundle embeds the full content of each changelog entry (`title`, `type`, `products`, and so on), plus a `file` block recording the source file name and checksum for provenance. Rendering — via the `{changelog}` directive, `changelog render`, or the CDN pipeline — never reads the original changelog files, so you can clean them up with `docs-builder changelog remove` immediately after bundling. +Every bundle embeds the full content of each changelog entry (`title`, `type`, `products`, and so on), plus a `file` block recording the source file name and checksum for provenance. Rendering — via the `{changelog}` directive, `changelog render`, or the CDN pipeline — never reads the original changelog files, so you can clean them up with `docs-builder changelog remove` immediately after bundling. To recreate changelog files later, use [`changelog unpack`](/cli/changelog/unpack.md). Unpack does not restore the original checksums. Public bundles produced by the upload scrubber are less likely to unpack because private PRs and issues may have been removed. When you bundle from a PR list or GitHub release and the command is sourcing from the CDN, it also adds uploaded changelogs whose `products[].versions` include that version (from `output_products`). Those files are appended to the same `entries` list; `rules.bundle` applies to them too. This automatic add does not apply to git-range bundles or `--force-local`. For more detail, see [Bundle changelogs](/data/release-notes/bundle.md). diff --git a/docs/cli/changelog/cmd-unpack.md b/docs/cli/changelog/cmd-unpack.md new file mode 100644 index 0000000000..d77f672dcf --- /dev/null +++ b/docs/cli/changelog/cmd-unpack.md @@ -0,0 +1,36 @@ +## Description + +Recreate individual changelog YAML files from a bundle. + +Each entry is written through the same path as [`changelog add`](/cli/changelog/add.md) (PR-anchored entries) or [`changelog note`](/cli/changelog/note.md) (entries with no PR, or products that include versions). The output is a new changelog YAML file, not a byte-for-byte copy of the original files. Bundle `file.checksum` values are provenance of the sourced YAML at bundle time and will not match the unpacked files. + +When you pass a full bundle file, its `.amend-*` files are merged first, the same way [`changelog render`](/cli/changelog/render.md) does. When you pass an amendment bundle file, only that file's `entries` are unpacked. `exclude-entries` are skipped; they are name and checksum stubs, not changelog files. + +:::{important} +The bundle argument must be a local `.yaml` or `.yml` file that exists on disk. + +If you download a bundle, get it from the private CDN instead of the public CDN. +Bundles in the public CDN have the private pull request and issue links removed. +::: + +## Filenames + +Filenames follow `changelog add` and `changelog note` rules, for example: + +- Add: `{pr}.yaml` (or `{pr}-{pr}.yaml` when one entry cites multiple PRs) +- Note: `note-{slug}.yml` + +If that name differs from the bundle provenance `file.name`, the command emits a warning and still writes the `add` or `note` name. + +## Examples + +```sh +docs-builder changelog unpack ./docs/releases/elasticsearch-serverless-2026-09-08.yaml \ + --output ./docs/changelog +``` + +```sh +docs-builder changelog unpack ./docs/releases/9.3.0.amend-1.yaml \ + --output ./docs/changelog \ + --concise +``` diff --git a/docs/cli/changelog/cmd-upload.md b/docs/cli/changelog/cmd-upload.md index c5e4f5fccf..2ae4652953 100644 --- a/docs/cli/changelog/cmd-upload.md +++ b/docs/cli/changelog/cmd-upload.md @@ -2,6 +2,8 @@ Upload changelog entries or bundle artifacts to S3 or Elasticsearch. The command discovers `.yaml` and `.yml` files in a local directory and uploads only files whose content hash changed since the last run. Changelog entries are uploaded once under `changelog/{org}/{repo}/{branch}/{file}`, keyed by the authoring owner, repository, and branch; bundles are uploaded under `bundle/{product}/{file}`, product-scoped from the bundle YAML. +A downstream scrubber copies published objects to the public bucket and removes pull request and issue links that are not on the allowlist (unlike bundle-time `# PRIVATE:` sentinels on the private side). Those public bundles are less likely to work with [`changelog unpack`](/cli/changelog/unpack.md). + To create bundles first, use [](/cli/changelog/bundle.md). For the end-to-end workflow, see [](/data/release-notes/bundle.md). @@ -79,7 +81,7 @@ Use `--artifact-type` to choose what to upload: Keying differs by artifact type: -- **Changelog entries** are uploaded **once** under the authoring owner/repo/branch, regardless of how many products they list (or none). The owner is resolved from `--owner`, then `bundle.owner` in `changelog.yml`, then the git remote origin; the repo from `--repo`, then `bundle.repo`, then the git remote origin; the branch from `--branch`, then the current checkout's branch. The branch is stored verbatim, so a branch name containing `/` (for example `feature/foo`) becomes additional key segments. +- **Changelog entries** are uploaded once under the authoring owner/repo/branch, regardless of how many products they list (or none). The owner is resolved from `--owner`, then `bundle.owner` in `changelog.yml`, then the git remote origin; the repo from `--repo`, then `bundle.repo`, then the git remote origin; the branch from `--branch`, then the current checkout's branch. The branch is stored verbatim, so a branch name containing `/` (for example `feature/foo`) becomes additional key segments. - **Bundles** are uploaded once per product listed in the bundle's `products[].product` field (a bundle that declares multiple products is written under each product prefix). Amend sidecars produced from a CDN parent (`changelog bundle-amend /bundle/{product}/{file}.yaml`) are uploaded like any other bundle YAML. ## Upload targets diff --git a/docs/cli/changelog/index.md b/docs/cli/changelog/index.md index 4f2e834ea3..8b03081135 100644 --- a/docs/cli/changelog/index.md +++ b/docs/cli/changelog/index.md @@ -3,7 +3,7 @@ The `changelog` commands manage a file-per-change workflow that produces release ## Typical workflow 1. **Configure** — create `docs/changelog.yml` with label mappings and bundle profiles: `docs-builder changelog init` -2. **Create** — add a changelog YAML for each notable PR: `docs-builder changelog add`. For items not tied to a PR (known issues, advisories), use `docs-builder changelog note`. +2. **Create** — add a changelog YAML for each notable PR: `docs-builder changelog add`. For items not tied to a PR (known issues, advisories), use `docs-builder changelog note`. To recreate files from an existing bundle, use `docs-builder changelog unpack`. 3. **Bundle** — aggregate entries for a release: `docs-builder changelog bundle` 4. **Publish** — render the bundle to a release notes page: `docs-builder changelog render` diff --git a/docs/data/release-notes/create.md b/docs/data/release-notes/create.md index cbe37ad566..b3b265db4b 100644 --- a/docs/data/release-notes/create.md +++ b/docs/data/release-notes/create.md @@ -66,10 +66,18 @@ If you already have automated release notes for GitHub releases, you can use the Any command strings that contain special characters (such as backquotes) must be preceded with a backslash escape character (`\`). ::: - For the most up-to-date command syntax, use the `-h` option or refer to [](/cli/changelog/add.md) and [](/cli/changelog/note.md). + For the most up-to-date command syntax, use the `-h` option or refer to [](/cli/changelog/add.md) and [](/cli/changelog/note.md). 1. [Review the output file](#review). +## Recreate changelog files from a bundle [unpack] + +If the original changelog files were deleted after bundling, you can recreate them from a bundle with [`changelog unpack`](/cli/changelog/unpack.md). The command maps each entry onto `changelog add` or `changelog note`; it does not restore original checksums or comments. + +:::{tip} +Use local bundles or bundles downloaded from the private CDN. Bundles scrubbed during [`changelog upload`](/cli/changelog/upload.md) drop private PRs and issues, which means the `unpack` command will generate incomplete changelogs. +::: + ## Create changelogs from GitHub actions [github-actions] For details about this method, refer to the [README](https://github.com/elastic/docs-actions/blob/main/changelog/README.md). diff --git a/src/Elastic.Documentation/ReleaseNotes/ChangelogTextUtilities.cs b/src/Elastic.Documentation/ReleaseNotes/ChangelogTextUtilities.cs index 8e081a822b..3e305c3927 100644 --- a/src/Elastic.Documentation/ReleaseNotes/ChangelogTextUtilities.cs +++ b/src/Elastic.Documentation/ReleaseNotes/ChangelogTextUtilities.cs @@ -239,6 +239,35 @@ public static bool TitleNeedsDefensiveYamlQuoting(string? title) private const string PrivateReferenceSentinelPrefix = "# PRIVATE:"; + /// + /// Unwraps a # PRIVATE: sentinel to the underlying PR or issue reference. + /// Returns the trimmed original when the value is not a sentinel. + /// + public static string? StripPrivateReferenceSentinel(string? reference) + { + if (string.IsNullOrWhiteSpace(reference)) + return null; + + var trimmed = reference.Trim(); + if (!trimmed.StartsWith(PrivateReferenceSentinelPrefix, StringComparison.OrdinalIgnoreCase)) + return trimmed; + + var underlying = trimmed[PrivateReferenceSentinelPrefix.Length..].Trim(); + return string.IsNullOrWhiteSpace(underlying) ? null : underlying; + } + + /// + /// Unwraps # PRIVATE: sentinels in a reference list. Empty after stripping are omitted. + /// + public static string[]? StripPrivateReferenceSentinels(IReadOnlyList? references) + { + if (references is not { Count: > 0 }) + return null; + + var stripped = references.Select(StripPrivateReferenceSentinel).Where(r => !string.IsNullOrWhiteSpace(r)).Select(r => r!).ToArray(); + return stripped.Length == 0 ? null : stripped; + } + /// /// Returns the first repository segment from a bundle string /// (e.g. elasticsearch+kibana → elasticsearch) for defaulting bare numeric PR/issue refs. diff --git a/src/services/Elastic.Changelog/Creation/ChangelogCreationService.cs b/src/services/Elastic.Changelog/Creation/ChangelogCreationService.cs index 27f3442ccf..3b4d46d5ef 100644 --- a/src/services/Elastic.Changelog/Creation/ChangelogCreationService.cs +++ b/src/services/Elastic.Changelog/Creation/ChangelogCreationService.cs @@ -214,6 +214,54 @@ public async Task CreateNote(IDiagnosticsCollector collector, CreateChange } } + /// + /// Writes a single changelog from fully populated fields without GitHub fetches or splitting + /// multi-PR inputs into one file per PR. Used by changelog unpack so a shipped bundle + /// entry round-trips as one add-shaped file. + /// + public async Task CreatePreparedChangelog(IDiagnosticsCollector collector, CreateChangelogArguments input, Cancel ctx) + { + try + { + var config = await _configLoader.LoadChangelogConfiguration(collector, input.Config, ctx); + if (config == null) + { + collector.EmitError(string.Empty, "Failed to load changelog configuration"); + return false; + } + + input = ApplyConfigDefaults(input, config) with { ExtractReleaseNotes = false, ExtractIssues = false }; + + if (input.Prs is { Length: > 1 }) + { + if (!_validator.ValidateMultiplePrFormat(collector, input.Prs, input.Owner, input.Repo)) + return false; + } + else if (!_validator.ValidatePrFormat(collector, input.Prs?.FirstOrDefault(), input.Owner, input.Repo)) + return false; + + if (input.Issues is { Length: > 1 }) + { + if (!_validator.ValidateMultipleIssueFormat(collector, input.Issues, input.Owner, input.Repo)) + return false; + } + else if (!_validator.ValidateIssueFormat(collector, input.Issues?.FirstOrDefault(), input.Owner, input.Repo)) + return false; + + return await WriteValidatedChangelog(collector, input, config, ctx); + } + catch (IOException ioEx) + { + collector.EmitError(string.Empty, $"IO error creating changelog: {ioEx.Message}", ioEx); + return false; + } + catch (UnauthorizedAccessException uaEx) + { + collector.EmitError(string.Empty, $"Access denied creating changelog: {uaEx.Message}", uaEx); + return false; + } + } + internal static CreateChangelogArguments ApplyConfigDefaults(CreateChangelogArguments input, ChangelogConfiguration config) => // Filename strategy is always Pr now; UsePrNumber is kept for backward compat but is effectively always true. input with @@ -390,7 +438,17 @@ private async Task CreateSingleChangelogAsync( else if (!string.IsNullOrWhiteSpace(prUrl)) _logger.LogInformation("All required fields already provided, skipping PR API fetch for {PrUrl}", prUrl); - // If still no products, fall back to products.default or repo name inference + return await WriteValidatedChangelog(collector, input, config, ctx, prFetchFailed); + } + + private async Task WriteValidatedChangelog( + IDiagnosticsCollector collector, + CreateChangelogArguments input, + ChangelogConfiguration config, + Cancel ctx, + bool prFetchFailed = false + ) + { if (input.Products.Count == 0) { var inferredProducts = InferProducts(config.ProductsConfiguration, input.Repo); @@ -398,19 +456,15 @@ private async Task CreateSingleChangelogAsync( input = input with { Products = inferredProducts }; } - // Validate required fields if (!_validator.ValidateRequiredFields(collector, input, prFetchFailed)) return false; - // Entries must not carry version targets; applicability comes from the origin branch if (!_validator.ValidateNoVersionTarget(collector, input)) return false; - // Validate against configuration if (!_validator.ValidateAgainstConfiguration(collector, input, config)) return false; - // Write changelog file return await _fileWriter.WriteChangelogAsync( collector, input, diff --git a/src/services/Elastic.Changelog/Creation/ChangelogFileWriter.cs b/src/services/Elastic.Changelog/Creation/ChangelogFileWriter.cs index 7226328d18..f8deb57008 100644 --- a/src/services/Elastic.Changelog/Creation/ChangelogFileWriter.cs +++ b/src/services/Elastic.Changelog/Creation/ChangelogFileWriter.cs @@ -74,7 +74,7 @@ public async Task WriteNoteAsync(CreateChangelogArguments input, Changelog if (!fileSystem.Directory.Exists(outputDir)) _ = fileSystem.Directory.CreateDirectory(outputDir); - var filename = GenerateNoteFilename(input.NoteName, input.Title); + var filename = GetNoteFileName(input.NoteName, input.Title); var filePath = fileSystem.Path.Join(outputDir, filename); var normalizedContent = ChangelogUtf8Normalization.StripLeadingUtf8BomChar(yamlContent); @@ -83,7 +83,7 @@ public async Task WriteNoteAsync(CreateChangelogArguments input, Changelog return true; } - private static string GenerateNoteFilename(string? noteName, string? title) + internal static string GetNoteFileName(string? noteName, string? title) { var source = !string.IsNullOrWhiteSpace(noteName) ? noteName : title; if (string.IsNullOrWhiteSpace(source)) @@ -116,29 +116,36 @@ private static string Slugify(string text) /// Maximum filename length before extension to avoid filesystem path-too-long errors. private const int MaxFilenameLength = 200; - private string? GenerateFilename(IDiagnosticsCollector collector, CreateChangelogArguments input) + internal static string? TryGetChangelogFileName(CreateChangelogArguments input) { - if (input.Prs is { Length: > 0 }) - { - var numbers = input - .Prs - .Select(pr => ChangelogTextUtilities.ExtractPrNumber(pr, input.Owner, input.Repo)) - .Where(n => n.HasValue) - .Select(n => n!.Value) - .Distinct() - .OrderBy(n => n) - .ToList(); - - if (numbers.Count > 0) - { - var joined = $"{string.Join("-", numbers)}.yaml"; - if (joined.Length <= MaxFilenameLength + 5) // ".yaml" = 5 chars + if (input.Prs is not { Length: > 0 }) + return null; - return joined; - // Too many PRs: use compact format to avoid path-too-long errors - return $"{numbers[0]}-to-{numbers[^1]}-{numbers.Count}-prs.yaml"; - } - } + var numbers = input + .Prs + .Select(pr => ChangelogTextUtilities.ExtractPrNumber(pr, input.Owner, input.Repo)) + .Where(n => n.HasValue) + .Select(n => n!.Value) + .Distinct() + .OrderBy(n => n) + .ToList(); + + if (numbers.Count == 0) + return null; + + var joined = $"{string.Join("-", numbers)}.yaml"; + if (joined.Length <= MaxFilenameLength + 5) // ".yaml" = 5 chars + + return joined; + + return $"{numbers[0]}-to-{numbers[^1]}-{numbers.Count}-prs.yaml"; + } + + private string? GenerateFilename(IDiagnosticsCollector collector, CreateChangelogArguments input) + { + var filename = TryGetChangelogFileName(input); + if (filename != null) + return filename; collector.EmitError( string.Empty, diff --git a/src/services/Elastic.Changelog/Creation/ChangelogUnpackService.cs b/src/services/Elastic.Changelog/Creation/ChangelogUnpackService.cs new file mode 100644 index 0000000000..ff6b87d45a --- /dev/null +++ b/src/services/Elastic.Changelog/Creation/ChangelogUnpackService.cs @@ -0,0 +1,269 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Changelog.Bundling; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.FileSystems; +using Elastic.Documentation.ReleaseNotes; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using YamlDotNet.Core; + +namespace Elastic.Changelog.Creation; + +/// Arguments for . +public record UnpackBundleArguments +{ + public required string BundleFile { get; init; } + public string? Output { get; init; } + public string? Config { get; init; } + public bool Concise { get; init; } +} + +/// +/// Recreates individual changelog YAML files from a bundle by mapping each entry onto +/// or +/// . +/// +public sealed class ChangelogUnpackService( + ILoggerFactory logFactory, + IChangelogFileSystem fileSystem, + IConfigurationContext configurationContext +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly ChangelogCreationService _creation = new(logFactory, configurationContext, fileSystem); + + private readonly record struct UnpackEntryContext(UnpackBundleArguments Input, string Owner, string? Repo); + + public async Task UnpackBundle(IDiagnosticsCollector collector, UnpackBundleArguments input, Cancel ctx) + { + if (!fileSystem.File.Exists(input.BundleFile)) + { + collector.EmitError(input.BundleFile, "Bundle file does not exist"); + return false; + } + + Bundle bundle; + try + { + var yaml = await fileSystem.File.ReadAllTextAsync(input.BundleFile, ctx); + bundle = ReleaseNotesSerialization.DeserializeBundle(yaml); + } + catch (YamlException yamlEx) + { + collector.EmitError(input.BundleFile, $"Failed to deserialize bundle file: {yamlEx.Message}", yamlEx); + return false; + } + + var entries = await ResolveEntries(collector, input.BundleFile, bundle, ctx); + if (entries == null) + return false; + + if (entries.Count == 0) + { + _logger.LogInformation("No changelog entries to unpack from {BundleFile}", input.BundleFile); + return true; + } + + var (owner, repo) = ResolveOwnerRepo(bundle); + var entryContext = new UnpackEntryContext(input, owner, repo); + var anyFailed = false; + foreach (var entry in entries) + { + if (!await UnpackEntry(collector, entryContext, entry, ctx)) + anyFailed = true; + } + + return !anyFailed && collector.Errors == 0; + } + + private async Task?> ResolveEntries( + IDiagnosticsCollector collector, + string bundleFile, + Bundle bundle, + Cancel ctx + ) + { + if (BundleAmendMerger.IsAmendFile(bundleFile)) + { + if (bundle.ExcludeEntries.Count > 0) + { + _logger.LogInformation( + "Skipping {Count} exclude-entries on amend sidecar {BundleFile}; exclusions cannot be recreated as changelog files", + bundle.ExcludeEntries.Count, + bundleFile + ); + } + + return bundle.Entries; + } + + var amendFiles = ChangelogBundleAmendService.DiscoverAmendFiles(fileSystem, bundleFile); + if (amendFiles.Count == 0) + return bundle.Entries; + + _logger.LogInformation("Found {Count} amend file(s) for bundle {BundleFile}", amendFiles.Count, bundleFile); + var amendBundles = new List(); + foreach (var amendFile in amendFiles) + { + try + { + var amendContent = await fileSystem.File.ReadAllTextAsync(amendFile, ctx); + var amendBundle = ReleaseNotesSerialization.DeserializeBundle(amendContent); + amendBundles.Add(amendBundle); + _logger.LogInformation( + "Merging amend file {AmendFile} ({AddCount} additions, {ExcludeCount} exclusions)", + amendFile, + amendBundle.Entries.Count, + amendBundle.ExcludeEntries.Count + ); + } + catch (YamlException yamlEx) + { + collector.EmitError(amendFile, $"Failed to deserialize amend file: {yamlEx.Message}", yamlEx); + return null; + } + } + + return BundleAmendMerger.MergeEntries(bundle.Entries, amendBundles); + } + + private async Task UnpackEntry(IDiagnosticsCollector collector, UnpackEntryContext context, BundledEntry entry, Cancel ctx) + { + if (IsMarker(entry)) + { + _logger.LogInformation("Skipping marker entry {FileName}", entry.File?.Name ?? ""); + return true; + } + + var mapped = MapToArguments(context, entry); + if (mapped == null) + { + collector.EmitError( + context.Input.BundleFile, + EntryLabel(entry) + " cannot be unpacked: after stripping private-link sentinels it has no pull-request " + + "references and its products have no versions. " + + "changelog add requires a PR; changelog note requires product versions. " + + "Bundles scrubbed during changelog upload often drop private PRs and issues — unpack the private-side bundle instead." + ); + return false; + } + + var (args, useNote) = mapped.Value; + EmitFilenameWarning(collector, entry, args, useNote); + + return useNote ? await _creation.CreateNote(collector, args, ctx) : await _creation.CreatePreparedChangelog(collector, args, ctx); + } + + private static (CreateChangelogArguments Args, bool UseNote)? MapToArguments(UnpackEntryContext context, BundledEntry entry) + { + var prs = ChangelogTextUtilities.StripPrivateReferenceSentinels(entry.Prs); + var issues = ChangelogTextUtilities.StripPrivateReferenceSentinels(entry.Issues); + var products = ToProductArguments(entry.Products); + var hasPrs = prs is { Length: > 0 }; + var hasVersions = products.Any(p => p.Versions.Count > 0); + + if (!hasPrs && !hasVersions) + return null; + + var useNote = !hasPrs || hasVersions; + var args = new CreateChangelogArguments + { + Title = entry.Title, + Type = entry.Type?.ToStringFast(true), + Subtype = entry.Subtype?.ToStringFast(true), + Products = products, + Areas = entry.Areas?.ToArray() ?? [], + Prs = prs, + Issues = issues, + Description = entry.Description, + Impact = entry.Impact, + Action = entry.Action, + FeatureId = entry.FeatureId, + Highlight = entry.Highlight, + Owner = context.Owner, + Repo = context.Repo, + Output = context.Input.Output, + Config = context.Input.Config, + Concise = context.Input.Concise, + ExtractReleaseNotes = false, + ExtractIssues = false, + UsePrNumber = true, + IsNote = useNote + }; + return (args, useNote); + } + + private static void EmitFilenameWarning( + IDiagnosticsCollector collector, + BundledEntry entry, + CreateChangelogArguments args, + bool useNote + ) + { + var provenance = LeafFileName(entry.File?.Name); + if (string.IsNullOrWhiteSpace(provenance)) + return; + + var expected = useNote + ? ChangelogFileWriter.GetNoteFileName(args.NoteName, args.Title) + : ChangelogFileWriter.TryGetChangelogFileName(args); + if (string.IsNullOrWhiteSpace(expected)) + return; + + if (string.Equals(provenance, expected, StringComparison.OrdinalIgnoreCase)) + return; + + collector.EmitWarning( + string.Empty, + $"Unpack will write '{expected}' for provenance file '{provenance}' using changelog {(useNote ? "note" : "add")} naming." + ); + } + + private static IReadOnlyList ToProductArguments(IReadOnlyList? products) + { + if (products is not { Count: > 0 }) + return []; + + return products.Select( + p => new ProductArgument + { + Product = p.ProductId, + Target = p.Versions.Count > 0 ? string.Join('|', p.Versions) : null, + Lifecycle = p.Lifecycle?.ToStringFast(true) + } + ).ToList(); + } + + private static (string Owner, string? Repo) ResolveOwnerRepo(Bundle bundle) + { + BundledProduct? product = null; + foreach (var candidate in bundle.Products) + { + if (string.IsNullOrWhiteSpace(candidate.Repo)) + continue; + product = candidate; + break; + } + + product ??= bundle.Products.Count > 0 ? bundle.Products[0] : null; + return (string.IsNullOrWhiteSpace(product?.Owner) ? "elastic" : product.Owner, product?.Repo); + } + + private static bool IsMarker(BundledEntry entry) => !string.IsNullOrWhiteSpace(entry.Link) && string.IsNullOrWhiteSpace(entry.Title); + + private static string EntryLabel(BundledEntry entry) => + !string.IsNullOrWhiteSpace(entry.File?.Name) ? $"Entry '{LeafFileName(entry.File.Name)}'" : $"Entry '{entry.Title ?? ""}'"; + + private static string? LeafFileName(string? name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.GetFileName(name.Replace('\\', '/')); + } +} diff --git a/src/tooling/docs-builder/Commands/ChangelogCommand.cs b/src/tooling/docs-builder/Commands/ChangelogCommand.cs index 01fa401177..f9936b052d 100644 --- a/src/tooling/docs-builder/Commands/ChangelogCommand.cs +++ b/src/tooling/docs-builder/Commands/ChangelogCommand.cs @@ -1573,6 +1573,52 @@ static async (s, collector, state, ctx) => await s.RenderChangelogs(collector, s return await serviceInvoker.InvokeAsync(ctx); } + /// Recreate individual changelog YAML files from a bundle using changelog add and changelog note. + /// + /// Writes one file per bundle entry through the same writers as changelog add (PR-anchored) + /// and changelog note (no PR, or products with versions). Filenames follow add and note rules, + /// not the bundle file.name provenance. Checksums are not reproduced. + /// Bundles that were scrubbed during changelog upload (public-bucket copies) have their + /// private PRs and issues removed, so they are less likely to unpack successfully. Prefer the + /// private-side bundle YAML. + /// + /// Local bundle or amend YAML file to unpack. The file must exist on disk; CDN locators such as /bundle/{product}/{file}.yaml are not accepted. A parent bundle is merged with sibling .amend-* bundles first (same as changelog render). An amend bundle unpacks only that file's entries; exclude-entries are skipped. + /// Path to changelog.yml. Defaults to docs/changelog.yml. Type and area validation uses this configuration. + /// Directory for written changelog files. Defaults to bundle.directory in changelog.yml, then the current directory. + /// Omit schema reference comments from generated YAML, matching changelog add --concise. + /// Cancellation token + [NoOptionsInjection] + public async Task Unpack( + [Argument, Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo bundle, + [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo? config = null, + string? output = null, + bool concise = false, + CancellationToken ct = default + ) + { + var ctx = ct; + await using var serviceInvoker = new ServiceInvoker(collector); + + var bundleConfig = await new ChangelogConfigurationLoader(logFactory, configurationContext, _fileSystem).LoadChangelogConfiguration( + collector, + config?.FullName, + ctx + ); + var resolvedOutput = !string.IsNullOrWhiteSpace(output) ? NormalizePath(output) : bundleConfig?.Bundle?.Directory; + + var service = new ChangelogUnpackService(logFactory, _fileSystem, configurationContext); + var input = new UnpackBundleArguments + { + BundleFile = bundle.FullName, + Output = resolvedOutput, + Config = config?.FullName, + Concise = concise + }; + + serviceInvoker.AddCommand(service, input, static async (s, c, state, token) => await s.UnpackBundle(c, state, token)); + return await serviceInvoker.InvokeAsync(ctx); + } + /// Create changelog entries from the PRs referenced in a GitHub release. /// Optional: GitHub repository in owner/repo format (e.g., "elastic/elasticsearch" or just "elasticsearch"). When omitted, falls back to bundle.repo in changelog.yml, then the GITHUB_REPOSITORY env var, then the git remote origin. /// Optional: Version tag to fetch (e.g., "v9.0.0", "9.0.0"). Defaults to "latest" diff --git a/tests/Elastic.Changelog.Tests/Changelogs/Create/ChangelogUnpackTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/Create/ChangelogUnpackTests.cs new file mode 100644 index 0000000000..00dd78cb73 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Changelogs/Create/ChangelogUnpackTests.cs @@ -0,0 +1,349 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Changelog.Creation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.ReleaseNotes; + +namespace Elastic.Changelog.Tests.Changelogs.Create; + +public class ChangelogUnpackTests(ITestOutputHelper output) : CreateChangelogTestBase(output) +{ + private const string ConfigYaml = + """ + pivot: + types: + feature: "type:feature" + enhancement: "type:enhancement" + bug-fix: "type:bug" + breaking-change: + known-issue: + lifecycles: + - preview + - beta + - ga + """; + + [Fact] + public async Task Unpack_PrEntry_WritesAddNamedFileAndStripsPrivateSentinel() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var bundlePath = await WriteBundle( + """ + products: + - product: cloud-serverless + target: 2026-09-08 + repo: elasticsearch-serverless + owner: elastic + entries: + - file: + name: 7606.yaml + checksum: 88dfd443adb87e70e71e7d8cb8417e4de1b91268 + type: enhancement + title: Add ECS user.domain to serverless audit logs + products: + - product: cloud-serverless + areas: + - Security + prs: + - '# PRIVATE: https://github.com/elastic/elasticsearch-serverless/pull/7606' + """ + ); + + var result = await Unpack(bundlePath, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + + var written = FileSystem.Path.Join(outputDir, "7606.yaml"); + FileSystem.File.Exists(written).Should().BeTrue(); + var yaml = await FileSystem.File.ReadAllTextAsync(written, TestContext.Current.CancellationToken); + yaml.Should().Contain("title: Add ECS user.domain to serverless audit logs"); + yaml.Should().Contain("type: enhancement"); + yaml.Should().Contain("https://github.com/elastic/elasticsearch-serverless/pull/7606"); + yaml.Should().NotContain("# PRIVATE:"); + yaml.Should().NotContain("checksum:"); + yaml.Should().Contain("##### Required fields"); + + var parsed = ReleaseNotesSerialization.DeserializeEntry(yaml); + parsed.Title.Should().Be("Add ECS user.domain to serverless audit logs"); + parsed.Type.Should().Be(ChangelogEntryType.Enhancement); + parsed.Prs.Should().ContainSingle().Which.Should().Be("https://github.com/elastic/elasticsearch-serverless/pull/7606"); + + ComputeSha1(yaml).Should().NotBe("88dfd443adb87e70e71e7d8cb8417e4de1b91268"); + } + + [Fact] + public async Task Unpack_VersionedEntryWithoutPrs_WritesNoteFile() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var bundlePath = await WriteBundle( + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + entries: + - file: + name: note-known-issue.yaml + checksum: deadbeef + type: known-issue + title: Alerts are not generated when flapping is off + products: + - product: elasticsearch + versions: + - 9.3.0 + """ + ); + + var result = await Unpack(bundlePath, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + Collector + .Diagnostics + .Should() + .Contain(d => d.Severity == Severity.Warning && d.Message.Contains("note-alerts-are-not-generated-when-flapping-is-off.yml")); + + var files = FileSystem.Directory.GetFiles(outputDir, "note-*.yml"); + files.Should().ContainSingle(); + var yaml = await FileSystem.File.ReadAllTextAsync(files[0], TestContext.Current.CancellationToken); + var parsed = ReleaseNotesSerialization.DeserializeEntry(yaml); + parsed.Type.Should().Be(ChangelogEntryType.KnownIssue); + parsed.Products.Should().ContainSingle().Which.Versions.Should().Contain("9.3.0"); + } + + [Fact] + public async Task Unpack_MultiPrEntry_WritesSingleFileNotOnePerPr() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var bundlePath = await WriteBundle( + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + entries: + - file: + name: 100-200.yaml + checksum: abc + type: bug-fix + title: Fix spanning two PRs + products: + - product: elasticsearch + prs: + - https://github.com/elastic/elasticsearch/pull/100 + - https://github.com/elastic/elasticsearch/pull/200 + """ + ); + + var result = await Unpack(bundlePath, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + FileSystem.Directory.GetFiles(outputDir, "*.yaml").Should().ContainSingle().Which.Should().EndWith("100-200.yaml"); + } + + [Fact] + public async Task Unpack_ParentMergesAmendAdditionAndExclusion() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var dir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString()); + FileSystem.Directory.CreateDirectory(dir); + var parent = FileSystem.Path.Join(dir, "release.yaml"); + await FileSystem.File.WriteAllTextAsync( + parent, + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + entries: + - file: + name: 111.yaml + checksum: keep + type: feature + title: Kept feature + products: + - product: elasticsearch + prs: + - https://github.com/elastic/elasticsearch/pull/111 + - file: + name: 222.yaml + checksum: drop + type: bug-fix + title: Retracted fix + products: + - product: elasticsearch + prs: + - https://github.com/elastic/elasticsearch/pull/222 + """, + TestContext.Current.CancellationToken + ); + await FileSystem.File.WriteAllTextAsync( + FileSystem.Path.Join(dir, "release.amend-1.yaml"), + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + exclude-entries: + - file: + name: 222.yaml + checksum: drop + entries: + - file: + name: 333.yaml + checksum: add + type: enhancement + title: Late addition + products: + - product: elasticsearch + prs: + - https://github.com/elastic/elasticsearch/pull/333 + """, + TestContext.Current.CancellationToken + ); + + var result = await Unpack(parent, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + var names = FileSystem.Directory.GetFiles(outputDir, "*.yaml").Select(FileSystem.Path.GetFileName).OrderBy(n => n).ToArray(); + names.Should().Equal("111.yaml", "333.yaml"); + } + + [Fact] + public async Task Unpack_AmendSidecar_WritesAdditionsOnly() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var dir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString()); + FileSystem.Directory.CreateDirectory(dir); + var amend = FileSystem.Path.Join(dir, "release.amend-1.yaml"); + await FileSystem.File.WriteAllTextAsync( + amend, + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + exclude-entries: + - file: + name: 222.yaml + checksum: drop + entries: + - file: + name: 333.yaml + checksum: add + type: enhancement + title: Late addition + products: + - product: elasticsearch + prs: + - https://github.com/elastic/elasticsearch/pull/333 + """, + TestContext.Current.CancellationToken + ); + + var result = await Unpack(amend, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + FileSystem.Directory.GetFiles(outputDir, "*.yaml").Select(FileSystem.Path.GetFileName).Should().Equal("333.yaml"); + } + + [Fact] + public async Task Unpack_ExcludeOnlyAmend_WritesNothing() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var dir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString()); + FileSystem.Directory.CreateDirectory(dir); + var amend = FileSystem.Path.Join(dir, "release.amend-2.yaml"); + await FileSystem.File.WriteAllTextAsync( + amend, + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch + owner: elastic + exclude-entries: + - file: + name: 222.yaml + checksum: drop + """, + TestContext.Current.CancellationToken + ); + + var result = await Unpack(amend, configPath, outputDir); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + FileSystem.Directory.Exists(outputDir).Should().BeFalse(); + } + + [Fact] + public async Task Unpack_ScrubbedEntryWithoutPrsOrVersions_Fails() + { + var configPath = await CreateConfigDirectory(ConfigYaml); + var outputDir = CreateOutputDirectory(); + var bundlePath = await WriteBundle( + """ + products: + - product: elasticsearch + target: 9.3.0 + repo: elasticsearch-serverless + owner: elastic + entries: + - file: + name: 7606.yaml + checksum: abc + type: enhancement + title: Scrubbed private change + products: + - product: elasticsearch + """ + ); + + var result = await Unpack(bundlePath, configPath, outputDir); + + result.Should().BeFalse(); + Collector.Errors.Should().BeGreaterThan(0); + Collector.Diagnostics.Should().Contain(d => d.Message.Contains("scrubbed during changelog upload")); + FileSystem.Directory.Exists(outputDir).Should().BeFalse(); + } + + private async Task Unpack(string bundlePath, string configPath, string outputDir) + { + var service = new ChangelogUnpackService(LoggerFactory, FileSystem, ConfigurationContext); + return await service.UnpackBundle( + Collector, + new UnpackBundleArguments { BundleFile = bundlePath, Config = configPath, Output = outputDir, Concise = false }, + TestContext.Current.CancellationToken + ); + } + + private async Task WriteBundle(string yaml) + { + var dir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString()); + FileSystem.Directory.CreateDirectory(dir); + var path = FileSystem.Path.Join(dir, "bundle.yaml"); + await FileSystem.File.WriteAllTextAsync(path, yaml, TestContext.Current.CancellationToken); + return path; + } +} diff --git a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/ChangelogTextUtilitiesTests.cs b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/ChangelogTextUtilitiesTests.cs index 7d0e20ff02..5a3415537a 100644 --- a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/ChangelogTextUtilitiesTests.cs +++ b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/ChangelogTextUtilitiesTests.cs @@ -228,4 +228,21 @@ public void HasVisibleLinks_WithHidePrivateLinks_ChecksCommentedFormat() result.Should().BeTrue(); } + + [Theory] + [InlineData("# PRIVATE: https://github.com/elastic/elasticsearch-serverless/pull/7606", "https://github.com/elastic/elasticsearch-serverless/pull/7606")] + [InlineData("https://github.com/elastic/elasticsearch/pull/1", "https://github.com/elastic/elasticsearch/pull/1")] + [InlineData(" # PRIVATE: elastic/repo#12 ", "elastic/repo#12")] + public void StripPrivateReferenceSentinel_UnwrapsSentinel(string input, string expected) => + ChangelogTextUtilities.StripPrivateReferenceSentinel(input).Should().Be(expected); + + [Fact] + public void StripPrivateReferenceSentinels_DropsEmptyAfterStrip() + { + var result = ChangelogTextUtilities.StripPrivateReferenceSentinels([ + "# PRIVATE: ", + "https://github.com/elastic/elasticsearch/pull/1" + ]); + result.Should().Equal("https://github.com/elastic/elasticsearch/pull/1"); + } }