diff --git a/docs/api/delivery.md b/docs/api/delivery.md index 5b8122d04..aac8eda73 100644 --- a/docs/api/delivery.md +++ b/docs/api/delivery.md @@ -20,6 +20,9 @@ Entities are content too, so their fields hold facts as stored: each value with | --- | --- | | `GET /api/v1/content/{type}` | The published entries of a content type, in the order they were created. | | `GET /api/v1/entries/{id}` | One published entry. | +| `GET /api/v1/entries/{id}.md` | The same entry as Markdown. | +| `GET /api/v1/index` | Every published entry with its type, title and URLs, as JSON. | +| `GET /api/v1/index.md` and `GET /llms.txt` | The same index as Markdown, grouped by content type. | `GET /api/v1/content/{type}` takes these query parameters: @@ -39,6 +42,7 @@ A list looks like this: "type": "product-page", "version": 3, "fields": { "product": "0192f3a1-0000-7000-8000-000000000001", "price": 99, "headline": "A sturdy drill" }, + "markdown": "/api/v1/entries/0192f3a1-7c2e-7a4b-9d3e-1f2a3b4c5d6e.md", "unresolved": [] } ], @@ -48,6 +52,22 @@ A list looks like this: `next` is `null` on the last page. Pages are keyed by id, so entries added while a client pages through do not shift the pages it has not read yet. +## Markdown and the site index + +Each published entry has a Markdown representation at `/api/v1/entries/{id}.md` (API-03). The URL is the entry's id, so it stays the same across versions and always shows the published version. + +- Front matter gives the id, type, version and URL, as quoted strings apart from the version number. +- The heading is the entry's title: its first text field that holds text. +- Each other field follows under its name, in the order of the content type definition that the published version was written against: + - rich text as written, since it is stored as Markdown, inside a block quote so that whatever it opens ends with its field; + - other text escaped where it could open structure, which means a value can never pose as a heading, a field, a link or HTML, while words and URLs keep their characters; + - lists as bullets, each item keeping its own paragraphs, and JSON in a fenced block; + - references to whole entries as links to their Markdown. +- On an entity, every fact, the title's included, is followed by a `**Source:**` paragraph, which also says when the fact is unsourced. A source that is a web address is a link. +- References that could not be resolved are listed under "Unresolved references", each with its reason. + +The site index lists every published entry for agents. `/llms.txt` serves it where agents look for it first. + ## Errors Errors are [RFC 9457](https://www.rfc-editor.org/rfc/rfc9457) problem details (`application/problem+json`). `errors` lists each error with a stable `code`: diff --git a/src/FluentCMS.Api/DeliveryEndpoints.cs b/src/FluentCMS.Api/DeliveryEndpoints.cs index 230f668d9..55bebd23c 100644 --- a/src/FluentCMS.Api/DeliveryEndpoints.cs +++ b/src/FluentCMS.Api/DeliveryEndpoints.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Text; using System.Text.Json; using FluentCMS.Core.Agents; using FluentCMS.Core.Content; @@ -16,19 +17,37 @@ public static class DeliveryEndpoints { public const string Prefix = "/api/v1"; + private const string Markdown = EntryMarkdown.MediaType + "; charset=utf-8"; + public static RouteGroupBuilder MapFluentCmsApi(this IEndpointRouteBuilder endpoints) { var api = endpoints.MapGroup(Prefix).AllowAnonymous(); api.MapGet("/content/{type}", ListAsync); api.MapGet("/entries/{id}", GetAsync); + api.MapGet("/entries/{id}.md", GetMarkdownAsync); + api.MapGet("/index", IndexAsync); + api.MapGet("/index.md", IndexMarkdownAsync); // Ids are parsed by the handlers and every other path under the prefix is answered here, so no API call falls through to the // admin's not-found page. api.MapGet("/{**path}", (string? path) => Problem( [new Error(DeliveryErrorCodes.EndpointNotFound, $"There is no delivery endpoint at {Prefix}/{path}; see docs/api/delivery.md.")])); + + // Where agents look for a site's index first, by the llms.txt convention. + endpoints.MapGet("/llms.txt", IndexMarkdownAsync).AllowAnonymous(); return api; } + public static string EntryUrl(Guid id) + { + return $"{Prefix}/entries/{id}"; + } + + public static string MarkdownUrl(Guid id) + { + return $"{Prefix}/entries/{id}.md"; + } + private static async Task ListAsync( string type, string? limit, @@ -69,6 +88,53 @@ private static async Task GetAsync(string id, IPublishedContent content return item.IsSuccess ? Results.Json(Item(item.Value)) : Problem(item.Errors); } + // The URL is the entry's id, so it stays the same across versions and always shows the published one (API-03). + private static async Task GetMarkdownAsync(string id, IPublishedContent content, CancellationToken cancellationToken) + { + var errors = new List(); + if (Id(id, "id", "the id of an entry", errors) is not { } entryId) + { + return Problem(errors); + } + + var item = await content.GetAsync(entryId, cancellationToken); + return item.IsSuccess + ? Results.Text(EntryMarkdown.Render(item.Value, MarkdownUrl(entryId), MarkdownUrl), Markdown) + : Problem(item.Errors); + } + + private static async Task IndexAsync(IPublishedContent content, CancellationToken cancellationToken) + { + var index = await content.IndexAsync(cancellationToken); + return Results.Json(new IndexResponse([.. index.Select(entry => new IndexEntryResponse( + entry.Id, + entry.ContentType, + entry.Version, + entry.Title, + EntryUrl(entry.Id), + MarkdownUrl(entry.Id)))])); + } + + private static async Task IndexMarkdownAsync(IPublishedContent content, CancellationToken cancellationToken) + { + var index = await content.IndexAsync(cancellationToken); + var markdown = new StringBuilder() + .Append("# Site index\n\n") + .Append(CultureInfo.InvariantCulture, $"Every published entry of this site, by content type, linked to its Markdown. The same index in JSON is at {Prefix}/index.\n"); + + foreach (var type in index.GroupBy(entry => (entry.ContentType, entry.ContentTypeName))) + { + markdown.Append(CultureInfo.InvariantCulture, $"\n## {EntryMarkdown.InlineText(type.Key.ContentTypeName)}\n\n"); + foreach (var entry in type) + { + var title = EntryMarkdown.InlineText(entry.Title ?? $"{entry.ContentTypeName} {entry.Id}"); + markdown.Append(CultureInfo.InvariantCulture, $"- [{title}]({MarkdownUrl(entry.Id)})\n"); + } + } + + return Results.Text(markdown.ToString(), Markdown); + } + private static Guid? Id(string? value, string name, string form, List errors) { if (value is null) @@ -92,6 +158,7 @@ private static ItemResponse Item(PublishedItem item) item.ContentType, item.Version, item.Fields, + MarkdownUrl(item.Id), [.. item.Unresolved.Select(reference => new UnresolvedResponse( reference.Path, reference.EntityId, @@ -120,7 +187,11 @@ private static IResult Problem(IReadOnlyList errors) private sealed record PageResponse(IReadOnlyList Items, Guid? Next); - private sealed record ItemResponse(Guid Id, string Type, int Version, JsonElement Fields, IReadOnlyList Unresolved); + private sealed record ItemResponse(Guid Id, string Type, int Version, JsonElement Fields, string Markdown, IReadOnlyList Unresolved); + + private sealed record IndexResponse(IReadOnlyList Items); + + private sealed record IndexEntryResponse(Guid Id, string Type, int Version, string? Title, string Url, string Markdown); private sealed record UnresolvedResponse(string Path, Guid EntityId, string Field, string Reason); } diff --git a/src/FluentCMS.Core/Delivery/EntryMarkdown.cs b/src/FluentCMS.Core/Delivery/EntryMarkdown.cs new file mode 100644 index 000000000..67a9083d3 --- /dev/null +++ b/src/FluentCMS.Core/Delivery/EntryMarkdown.cs @@ -0,0 +1,277 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using FluentCMS.Core.Content; +using FluentCMS.Core.Schema; + +namespace FluentCMS.Core.Delivery; + +// The Markdown representation of a published entry (API-03). Rich text is stored as Markdown (SCH-02), so it goes in as written, but +// quoted, so whatever it opens ends with its field. Every other value is written as text that reads as written. No value can pose +// as a heading, a field, a source or a link outside its own field. Fields follow in the order of the definition the published +// version was written against. +public static class EntryMarkdown +{ + public const string MediaType = "text/markdown"; + + // Markdown lines end in \n whatever the platform, JSON blocks included. + private static readonly JsonSerializerOptions Indented = new() { WriteIndented = true, NewLine = "\n" }; + + // The first text field that holds text, read from its fact on an entity, on one line; null when there is none. + public static string? Title(JsonElement fields, ContentTypeDefinition definition) + { + ArgumentNullException.ThrowIfNull(definition); + + return TitleField(fields, definition)?.Text; + } + + // Plain text as Markdown that reads as written; CommonMark lets a backslash escape any punctuation. Escaped wherever they are: + // what opens code, emphasis, links, autolinks, HTML and strikethrough, an underscore that is not inside a word, and an ampersand + // that starts an entity. Escaped where a line starts: what opens a heading, a list, a quote, a table, a heading underline, or the + // math and figure blocks of common extensions. + // Lines lose their indentation, so none becomes a code block; breaks stay. URLs and words keep their characters. + public static string Text(string text) + { + ArgumentNullException.ThrowIfNull(text); + + var escaped = new StringBuilder(text.Length + 16); + var lines = text.ReplaceLineEndings("\n").Split('\n'); + for (var index = 0; index < lines.Length; index++) + { + if (index > 0) + { + escaped.Append('\n'); + } + + var line = lines[index].TrimStart(); + var digits = line.TakeWhile(char.IsAsciiDigit).Count(); + for (var position = 0; position < line.Length; position++) + { + var character = line[position]; + if (NeedsEscape(line, position, digits)) + { + escaped.Append('\\'); + } + + escaped.Append(character); + } + } + + return escaped.ToString(); + } + + // Text for one line, such as a heading or a link's text: its whitespace collapsed, then escaped as Text is. + public static string InlineText(string text) + { + ArgumentNullException.ThrowIfNull(text); + + return Text(Line(text)); + } + + // EntryUrl gives the Markdown URL of another entry, for references to whole entries. + public static string Render(PublishedItem item, string url, Func entryUrl) + { + ArgumentNullException.ThrowIfNull(item); + ArgumentNullException.ThrowIfNull(entryUrl); + + var definition = item.Definition; + var title = TitleField(item.Fields, definition); + var markdown = new StringBuilder() + .Append("---\n") + .Append(CultureInfo.InvariantCulture, $"id: \"{item.Id}\"\n") + .Append(CultureInfo.InvariantCulture, $"type: \"{item.ContentType}\"\n") + .Append(CultureInfo.InvariantCulture, $"version: {item.Version}\n") + .Append(CultureInfo.InvariantCulture, $"url: \"{url}\"\n") + .Append("---\n\n") + .Append(CultureInfo.InvariantCulture, $"# {InlineText(title?.Text ?? $"{item.ContentTypeName} {item.Id}")}\n"); + + // The heading shows the title, so its field is not repeated below it, unless it is a fact, whose source belongs with it. + foreach (var field in definition.Fields.Where(field => definition.IsEntity || field.Name != title?.Field)) + { + if (Value(item.Fields, field.Name, definition.IsEntity) is not { } value) + { + continue; + } + + markdown.Append(CultureInfo.InvariantCulture, $"\n## {field.Name}\n\n").Append(Block(field.Type, field.Settings, value, entryUrl)).Append('\n'); + + // A paragraph with a strong label after the value, which escaped text cannot form and quoted rich text cannot reach. + if (definition.IsEntity) + { + markdown.Append(CultureInfo.InvariantCulture, $"\n**Source:** {Source(item.Fields, field.Name) ?? "none, so this fact is unsourced"}\n"); + } + } + + if (item.Unresolved.Count > 0) + { + markdown.Append("\n## Unresolved references\n\n"); + foreach (var reference in item.Unresolved) + { + markdown.Append(CultureInfo.InvariantCulture, $"- `{reference.Path}`: {reference.Field} of entry {reference.EntityId} {Reason(reference.Reason)}\n"); + } + } + + return markdown.ToString(); + } + + private static bool NeedsEscape(string line, int position, int digits) + { + var character = line[position]; + if (character is '\\' or '`' or '*' or '[' or ']' or '<' or '>' or '~') + { + return true; + } + + if (position == 0 && character is '#' or '-' or '+' or '=' or '|' or ':' or '$' or '^') + { + return true; + } + + if (position == digits && digits > 0 && character is '.' or ')') + { + return true; + } + + // A run of hashes that ends a line after a space would close a heading and be dropped from it. + if (character == '#' && char.IsWhiteSpace(line[position - 1]) && line[position..].TrimEnd().All(each => each == '#')) + { + return true; + } + + // An underscore inside a word opens no emphasis, as in a URL or an identifier. + if (character == '_') + { + return position == 0 || position == line.Length - 1 || !char.IsLetterOrDigit(line[position - 1]) || !char.IsLetterOrDigit(line[position + 1]); + } + + // Only an ampersand that starts a named or numeric entity reads as anything but itself. + if (character == '&') + { + var rest = line.AsSpan(position + 1); + var name = rest.StartsWith("#") ? rest[1..] : rest; + var length = 0; + while (length < name.Length && char.IsAsciiLetterOrDigit(name[length])) + { + length++; + } + + return length > 0 && length < name.Length && name[length] == ';'; + } + + return false; + } + + private static (string Field, string Text)? TitleField(JsonElement fields, ContentTypeDefinition definition) + { + foreach (var field in definition.Fields.Where(field => field.Type == "text")) + { + if (Value(fields, field.Name, definition.IsEntity) is { ValueKind: JsonValueKind.String } value) + { + return (field.Name, Line(value.GetString()!)); + } + } + + return null; + } + + // The field's value: on an entity, the value inside its fact. Null, blank and missing values are left out. + private static JsonElement? Value(JsonElement fields, string name, bool isEntity) + { + if (fields.ValueKind != JsonValueKind.Object || !fields.TryGetProperty(name, out var value)) + { + return null; + } + + if (isEntity && value.ValueKind == JsonValueKind.Object && !value.TryGetProperty(FactMetadata.Value, out value)) + { + return null; + } + + return value.ValueKind == JsonValueKind.Null || (value.ValueKind == JsonValueKind.String && string.IsNullOrWhiteSpace(value.GetString())) + ? null + : value; + } + + // A source that is an absolute web address is written as an autolink, which keeps it exact; anything else is escaped text. + private static string? Source(JsonElement fields, string name) + { + if (!fields.TryGetProperty(name, out var fact) + || fact.ValueKind != JsonValueKind.Object + || !fact.TryGetProperty(FactMetadata.Source, out var source) + || source.ValueKind != JsonValueKind.Object + || !source.TryGetProperty(FactMetadata.SourceReference, out var reference) + || reference.ValueKind != JsonValueKind.String) + { + return null; + } + + var address = reference.GetString()!.Trim(); + var text = Uri.TryCreate(address, UriKind.Absolute, out var uri) + && uri.Scheme is "http" or "https" + && !address.Any(character => char.IsWhiteSpace(character) || char.IsControl(character) || character is '<' or '>') + ? $"<{address}>" + : InlineText(address); + return source.TryGetProperty(FactMetadata.SourceKind, out var kind) && kind.ValueKind == JsonValueKind.String + ? $"{text} \\({InlineText(kind.GetString()!)}\\)" + : text; + } + + private static string Reason(UnresolvedReason reason) + { + return reason switch + { + UnresolvedReason.NotPublished => "is not published", + UnresolvedReason.NoValue => "has no value", + UnresolvedReason.Deleted => "is deleted", + _ => "cannot be resolved", + }; + } + + private static string Block(string type, JsonElement settings, JsonElement value, Func entryUrl) + { + if (type == "list" && value.ValueKind == JsonValueKind.Array) + { + var itemType = settings.ValueKind == JsonValueKind.Object && settings.TryGetProperty("itemType", out var item) && item.ValueKind == JsonValueKind.String + ? item.GetString()! + : ""; + var itemSettings = settings.ValueKind == JsonValueKind.Object && settings.TryGetProperty("itemSettings", out var nested) ? nested : FieldDefinition.EmptySettings; + + // Each item keeps its own block, indented under its bullet so paragraphs and fences stay inside the item. + return string.Join('\n', value.EnumerateArray() + .Where(element => element.ValueKind != JsonValueKind.Null) + .Select(element => "- " + Block(itemType, itemSettings, element, entryUrl).TrimStart().Replace("\n", "\n ", StringComparison.Ordinal).Replace("\n \n", "\n\n", StringComparison.Ordinal))); + } + + // A reference to an entity field already holds the resolved value; a reference to a whole entry holds its id. + var wholeEntry = type == "reference" + && !(settings.ValueKind == JsonValueKind.Object && settings.TryGetProperty("field", out var field) && field.ValueKind == JsonValueKind.String); + + return value.ValueKind switch + { + JsonValueKind.String when type == "richText" => Quoted(value.GetString()!.Trim()), + JsonValueKind.String when wholeEntry && value.TryGetGuid(out var id) => $"[{id}]({entryUrl(id)})", + JsonValueKind.String when type == "media" && value.TryGetGuid(out var asset) => $"Media asset `{asset}`", + JsonValueKind.String => Text(value.GetString()!), + JsonValueKind.Object or JsonValueKind.Array => Fenced(value), + _ when type == "json" => Fenced(value), + _ => value.GetRawText(), + }; + } + + // A block quote is a container, so a fence, an HTML block or a list the rich text leaves open closes where the quote ends. + private static string Quoted(string markdown) + { + return string.Join('\n', markdown.ReplaceLineEndings("\n").Split('\n').Select(line => line.Length == 0 ? ">" : $"> {line}")); + } + + // The serializer writes a backtick in a string as `, so no value inside can close the fence. + private static string Fenced(JsonElement value) + { + return $"```json\n{JsonSerializer.Serialize(value, Indented)}\n```"; + } + + private static string Line(string text) + { + return string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + } +} diff --git a/src/FluentCMS.Core/Delivery/IPublishedContent.cs b/src/FluentCMS.Core/Delivery/IPublishedContent.cs index 8ebe61661..f4a8123a1 100644 --- a/src/FluentCMS.Core/Delivery/IPublishedContent.cs +++ b/src/FluentCMS.Core/Delivery/IPublishedContent.cs @@ -1,6 +1,7 @@ using System.Text.Json; using FluentCMS.Core.Content; using FluentCMS.Core.Results; +using FluentCMS.Core.Schema; namespace FluentCMS.Core.Delivery; @@ -13,6 +14,9 @@ public interface IPublishedContent Task> ListAsync(PublishedQuery query, CancellationToken cancellationToken = default); Task> GetAsync(Guid id, CancellationToken cancellationToken = default); + + // Every published entry of every type the caller may read, by type and then by id, for the site index (API-03). + Task> IndexAsync(CancellationToken cancellationToken = default); } // After is the Next of the previous page. References keeps the entries whose published version points at that entry, entity or @@ -27,7 +31,18 @@ public sealed record PublishedQuery(string ContentType, int Limit = PublishedQue // Next is null on the last page. public sealed record PublishedPage(IReadOnlyList Items, Guid? Next); -public sealed record PublishedItem(Guid Id, string ContentType, int Version, JsonElement Fields, IReadOnlyList Unresolved); +// Title is EntryMarkdown.Title of the published version, null when it has no text. +public sealed record PublishedSummary(Guid Id, string ContentType, string ContentTypeName, int Version, string? Title); + +// The content type's name and the definition the version was written against come along, since the version is read against it. +public sealed record PublishedItem( + Guid Id, + string ContentType, + string ContentTypeName, + ContentTypeDefinition Definition, + int Version, + JsonElement Fields, + IReadOnlyList Unresolved); public static class DeliveryErrorCodes { diff --git a/src/FluentCMS.Infrastructure/Delivery/PublishedContentReader.cs b/src/FluentCMS.Infrastructure/Delivery/PublishedContentReader.cs index 43bc58ad3..0e8ffa7b1 100644 --- a/src/FluentCMS.Infrastructure/Delivery/PublishedContentReader.cs +++ b/src/FluentCMS.Infrastructure/Delivery/PublishedContentReader.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using FluentCMS.Core.Agents; using FluentCMS.Core.Content; using FluentCMS.Core.Delivery; @@ -10,7 +11,8 @@ namespace FluentCMS.Infrastructure.Delivery; -// An agent that reads through here keeps to its scope (MCP-05); people and anonymous readers see every published entry. +// An agent that reads through here keeps to its scope (MCP-05); people and anonymous readers see every published entry. Each item +// carries the definition its version was written against, so a revision of the type waiting for review changes nothing it shows. internal sealed class PublishedContentReader(FluentCmsDbContext context, IAgentPermissions permissions) : IPublishedContent { // Until multi-site (MST-01), every entry belongs to the one site (SCH-07). @@ -33,19 +35,19 @@ public async Task> ListAsync(PublishedQuery query, Cancell return Result.Failure(allowed.Errors); } - var typeId = await context.ContentTypes + var type = await context.ContentTypes .AsNoTracking() .Where(row => row.SiteId == SiteId && row.Slug == query.ContentType) - .Select(row => (Guid?)row.Id) + .Select(row => new { row.Id, row.Name }) .SingleOrDefaultAsync(cancellationToken); - if (typeId is null) + if (type is null) { return Result.Failure(SchemaErrorCodes.ContentTypeNotFound, $"There is no content type '{query.ContentType}' in this site."); } var entries = context.Entries .AsNoTracking() - .Where(entry => entry.SiteId == SiteId && entry.ContentTypeId == typeId && entry.PublishedVersion != null && entry.DeletedAt == null); + .Where(entry => entry.SiteId == SiteId && entry.ContentTypeId == type.Id && entry.PublishedVersion != null && entry.DeletedAt == null); // Ids are version 7, so they sort in the order of creation and a page boundary does not move as entries are added. if (query.After is { } after) @@ -59,20 +61,24 @@ public async Task> ListAsync(PublishedQuery query, Cancell .Any(reference => reference.EntryId == entry.Id && reference.Version == entry.PublishedVersion && reference.TargetId == target)); } - var rows = await entries - .OrderBy(entry => entry.Id) + var rows = await ( + from entry in entries + join version in context.EntryVersions.AsNoTracking() + on new { EntryId = entry.Id, Version = entry.PublishedVersion!.Value } equals new { version.EntryId, version.Version } + orderby entry.Id + select new { entry.Id, version.Version, version.ContentTypeVersion }) .Take(query.Limit + 1) - .Select(entry => new { entry.Id, Version = entry.PublishedVersion!.Value }) .ToListAsync(cancellationToken); var page = rows.Take(query.Limit).ToList(); + var definitions = await DefinitionsAsync([.. page.Select(row => (type.Id, row.ContentTypeVersion))], cancellationToken); var resolved = await EntryResolution.ResolveAsync( context, [.. page.Select(row => new VersionToResolve(row.Id, query.ContentType, row.Version))], cancellationToken); return Result.Success(new PublishedPage( - [.. resolved.Select(Item)], + [.. resolved.Select((entry, index) => Item(entry, type.Name, definitions[(type.Id, page[index].ContentTypeVersion)]))], rows.Count > query.Limit ? page[^1].Id : null)); } @@ -81,8 +87,10 @@ public async Task> GetAsync(Guid id, CancellationToken can var entry = await ( from row in context.Entries.AsNoTracking() join type in context.ContentTypes.AsNoTracking() on row.ContentTypeId equals type.Id + join version in context.EntryVersions.AsNoTracking() + on new { EntryId = row.Id, Version = row.PublishedVersion!.Value } equals new { version.EntryId, version.Version } where row.Id == id && row.SiteId == SiteId && row.PublishedVersion != null && row.DeletedAt == null - select new { type.Slug, Version = row.PublishedVersion!.Value }) + select new { row.ContentTypeId, type.Slug, type.Name, version.Version, version.ContentTypeVersion }) .SingleOrDefaultAsync(cancellationToken); if (entry is null) { @@ -95,12 +103,76 @@ join type in context.ContentTypes.AsNoTracking() on row.ContentTypeId equals typ return Result.Failure(allowed.Errors); } + var definitions = await DefinitionsAsync([(entry.ContentTypeId, entry.ContentTypeVersion)], cancellationToken); var resolved = await EntryResolution.ResolveAsync(context, [new VersionToResolve(id, entry.Slug, entry.Version)], cancellationToken); - return Result.Success(Item(resolved[0])); + return Result.Success(Item(resolved[0], entry.Name, definitions[(entry.ContentTypeId, entry.ContentTypeVersion)])); } - private static PublishedItem Item(ResolvedEntry entry) + public async Task> IndexAsync(CancellationToken cancellationToken = default) { - return new PublishedItem(entry.EntryId, entry.ContentType, entry.Version, entry.Fields, entry.Unresolved); + // The stored documents, not resolved views: a title is a text field of the entry itself, never a resolved reference. + var rows = await ( + from entry in context.Entries.AsNoTracking() + join type in context.ContentTypes.AsNoTracking() on entry.ContentTypeId equals type.Id + join version in context.EntryVersions.AsNoTracking() + on new { EntryId = entry.Id, Version = entry.PublishedVersion!.Value } equals new { version.EntryId, version.Version } + where entry.SiteId == SiteId && entry.PublishedVersion != null && entry.DeletedAt == null + select new { entry.Id, entry.ContentTypeId, type.Slug, type.Name, version.Version, version.ContentTypeVersion, version.Fields }) + .ToListAsync(cancellationToken); + + var readable = await permissions.ReadableTypesAsync(cancellationToken); + var shown = rows.Where(row => readable(row.Slug)).ToList(); + var definitions = await DefinitionsAsync([.. shown.Select(row => (row.ContentTypeId, row.ContentTypeVersion))], cancellationToken); + + return shown + .Select(row => + { + using var fields = JsonDocument.Parse(row.Fields); + return new PublishedSummary( + row.Id, + row.Slug, + row.Name, + row.Version, + EntryMarkdown.Title(fields.RootElement, definitions[(row.ContentTypeId, row.ContentTypeVersion)])); + }) + .OrderBy(summary => summary.ContentType, StringComparer.Ordinal) + .ThenBy(summary => summary.Id) + .ToList(); + } + + // The definitions of the type versions given, read in one query; every pair must be stored. + private async Task> DefinitionsAsync( + IReadOnlyCollection<(Guid TypeId, int Version)> wanted, + CancellationToken cancellationToken) + { + var pairs = wanted.ToHashSet(); + var typeIds = pairs.Select(pair => pair.TypeId).Distinct().ToList(); + var versions = pairs.Select(pair => pair.Version).Distinct().ToList(); + + var rows = await context.ContentTypeVersions + .AsNoTracking() + .Where(row => typeIds.Contains(row.ContentTypeId) && versions.Contains(row.Version)) + .Select(row => new { row.ContentTypeId, row.Version, row.Definition }) + .ToListAsync(cancellationToken); + + return rows + .Where(row => pairs.Contains((row.ContentTypeId, row.Version))) + .ToDictionary(row => (row.ContentTypeId, row.Version), row => ReadDefinition(row.ContentTypeId, row.Version, row.Definition)); + } + + private static ContentTypeDefinition ReadDefinition(Guid typeId, int version, string json) + { + var definition = ContentTypeDefinition.Parse(json); + + return definition.IsSuccess + ? definition.Value + : throw new InvalidOperationException( + $"Version {version} of content type {typeId} is stored in a form that cannot be read: " + + string.Join("; ", definition.Errors.Select(error => error.Message))); + } + + private static PublishedItem Item(ResolvedEntry entry, string typeName, ContentTypeDefinition definition) + { + return new PublishedItem(entry.EntryId, entry.ContentType, typeName, definition, entry.Version, entry.Fields, entry.Unresolved); } } diff --git a/tests/FluentCMS.IntegrationTests/Delivery/DeliveryEndpointsTests.cs b/tests/FluentCMS.IntegrationTests/Delivery/DeliveryEndpointsTests.cs index 6fb0bf28b..e29b0294a 100644 --- a/tests/FluentCMS.IntegrationTests/Delivery/DeliveryEndpointsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Delivery/DeliveryEndpointsTests.cs @@ -32,9 +32,43 @@ public async Task A_list_returns_items_with_their_resolved_fields_and_the_next_c Assert.Equal(99, (int)item["fields"]!["price"]!); var unresolved = Assert.Single(item["unresolved"]!.AsArray())!; Assert.Equal(("/spec", "notPublished"), ((string?)unresolved["path"], (string?)unresolved["reason"])); + Assert.Equal($"/api/v1/entries/{Page}.md", (string?)item["markdown"]); Assert.Equal(Page.ToString(), (string?)response["next"]); } + [Fact] + public async Task An_entry_reads_as_markdown_at_its_id() + { + await using var api = await StubApi.StartAsync(new StubContent()); + + var response = await api.Client.GetAsync($"/api/v1/entries/{Page}.md"); + var missing = await api.Client.GetAsync($"/api/v1/entries/{Guid.CreateVersion7()}.md"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType); + var markdown = await response.Content.ReadAsStringAsync(); + Assert.StartsWith($"---\nid: \"{Page}\"\ntype: \"product-page\"\nversion: 3\nurl: \"/api/v1/entries/{Page}.md\"\n---\n\n# A sturdy drill\n", markdown, StringComparison.Ordinal); + Assert.Contains($"[{Product}](/api/v1/entries/{Product}.md)", markdown, StringComparison.Ordinal); + await AssertProblemAsync(missing, HttpStatusCode.NotFound, ContentErrorCodes.EntryNotFound); + } + + [Fact] + public async Task The_site_index_lists_every_entry_in_json_markdown_and_llms_txt() + { + await using var api = await StubApi.StartAsync(new StubContent()); + + var json = await api.Client.GetFromJsonAsync("/api/v1/index"); + var markdown = await api.Client.GetStringAsync("/api/v1/index.md"); + var llms = await api.Client.GetStringAsync("/llms.txt"); + + var entries = json!["items"]!.AsArray(); + Assert.Equal([Product.ToString(), Page.ToString()], entries.Select(entry => (string?)entry!["id"])); + Assert.Equal(("Drill [cordless]", $"/api/v1/entries/{Product}", $"/api/v1/entries/{Product}.md"), ((string?)entries[0]!["title"], (string?)entries[0]!["url"], (string?)entries[0]!["markdown"])); + Assert.Contains($@"## Product{"\n\n"}- [Drill \[cordless\]](/api/v1/entries/{Product}.md){"\n"}", markdown, StringComparison.Ordinal); + Assert.Contains($"## Product page\n\n- [Product page {Page}](/api/v1/entries/{Page}.md)\n", markdown, StringComparison.Ordinal); + Assert.Equal(markdown, llms); + } + [Fact] public async Task An_entry_reads_by_id_and_a_missing_one_is_a_problem_with_404() { @@ -84,6 +118,7 @@ public async Task The_host_serves_the_api_to_anonymous_readers_from_the_database // The admin's not-found page answers every other path of the host, so the API answers each of its own. [Theory] [InlineData("/api/v1/entries/not-a-guid", HttpStatusCode.BadRequest, DeliveryErrorCodes.ParameterInvalid)] + [InlineData("/api/v1/entries/not-a-guid.md", HttpStatusCode.BadRequest, DeliveryErrorCodes.ParameterInvalid)] [InlineData("/api/v1/entries/", HttpStatusCode.NotFound, DeliveryErrorCodes.EndpointNotFound)] [InlineData("/api/v1/content/", HttpStatusCode.NotFound, DeliveryErrorCodes.EndpointNotFound)] [InlineData("/api/v1/nothing/here", HttpStatusCode.NotFound, DeliveryErrorCodes.EndpointNotFound)] @@ -124,13 +159,29 @@ public Task> GetAsync(Guid id, CancellationToken cancellat : Result.Failure(ContentErrorCodes.EntryNotFound, $"There is no published entry {id} in this site.")); } + public Task> IndexAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult>( + [ + new PublishedSummary(Product, "product", "Product", 1, "Drill [cordless]"), + new PublishedSummary(Page, "product-page", "Product page", 3, null), + ]); + } + private static PublishedItem Item() { return new PublishedItem( Page, "product-page", + "Product page", + new ContentTypeDefinition( + [ + new FieldDefinition("product", "reference", settings: JsonSerializer.Deserialize("""{"target": "product"}""")), + new FieldDefinition("price", "number"), + new FieldDefinition("headline", "text"), + ]), 3, - JsonSerializer.Deserialize($$"""{"product": "{{Product}}", "price": 99, "spec": null}"""), + JsonSerializer.Deserialize($$"""{"product": "{{Product}}", "price": 99, "spec": null, "headline": "A sturdy drill"}"""), [new UnresolvedReference("/spec", Guid.CreateVersion7(), "value", UnresolvedReason.NotPublished)]); } } diff --git a/tests/FluentCMS.IntegrationTests/Delivery/PublishedContentTests.cs b/tests/FluentCMS.IntegrationTests/Delivery/PublishedContentTests.cs index b49951b77..32bc6aece 100644 --- a/tests/FluentCMS.IntegrationTests/Delivery/PublishedContentTests.cs +++ b/tests/FluentCMS.IntegrationTests/Delivery/PublishedContentTests.cs @@ -7,6 +7,7 @@ using FluentCMS.Infrastructure.Persistence; using FluentCMS.IntegrationTests.ChangeSets; using FluentCMS.IntegrationTests.Content; +using FluentCMS.IntegrationTests.Schema; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using static FluentCMS.IntegrationTests.Content.EntryScenario; @@ -157,6 +158,90 @@ await scenario.PublishAsync( Assert.Equal(AgentPermissionErrorCodes.ScopeDenied, Code(await content.GetAsync(page))); Assert.Equal(AgentPermissionErrorCodes.ScopeDenied, Code(await content.ListAsync(new PublishedQuery("page")))); Assert.Equal([product], (await content.ListAsync(new PublishedQuery("product"))).Value.Items.Select(item => item.Id)); + Assert.Equal([product], (await content.IndexAsync()).Select(summary => summary.Id)); + } + + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task The_index_lists_each_published_entry_once_with_the_title_of_its_published_version(string provider) + { + await using var scenario = await CreateAsync(provider); + var product = await ProductAsync(scenario, "99"); + var page = await PageAsync(scenario, product, "First headline"); + var draft = await PageAsync(scenario, product, "Draft"); + var deleted = await PageAsync(scenario, product, "Deleted"); + var post = (await scenario.CreateAsync("blog-post", """{"title": "Launch"}""")).Value.Id; + foreach (var entry in new[] { product, page, deleted, post }) + { + await PublishAsync(scenario, entry, 1); + } + + await scenario.RunAsync(entries => entries.DeleteAsync(deleted, 1)); + await scenario.UpdateAsync(page, 1, $$"""{"product": "{{product}}", "price": "{{product}}", "headline": "Second headline"}"""); + + var before = await ContentAsync(scenario, content => content.IndexAsync()); + await PublishAsync(scenario, page, 2); + var after = await ContentAsync(scenario, content => content.IndexAsync()); + + Assert.Equal( + [ + new PublishedSummary(post, "blog-post", "Blog Post", 1, "Launch"), + new PublishedSummary(product, "product", "Product", 1, "Drill"), + new PublishedSummary(page, "product-page", "Product Page", 1, "First headline"), + ], + before); + Assert.Equal((2, "Second headline"), (after[^1].Version, after[^1].Title)); + Assert.DoesNotContain(after, summary => summary.Id == draft || summary.Id == deleted); + } + + // API-03: the Markdown URL is the entry's id, so it stays the same across versions and always shows the published one. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task The_markdown_of_a_published_page_follows_its_published_version(string provider) + { + await using var scenario = await CreateAsync(provider); + var product = await ProductAsync(scenario, "99"); + await PublishAsync(scenario, product, 1); + var page = await PageAsync(scenario, product, "A sturdy drill"); + await PublishAsync(scenario, page, 1); + await scenario.UpdateAsync(page, 1, $$"""{"product": "{{product}}", "price": "{{product}}", "headline": "Now cheaper"}"""); + + async Task MarkdownAsync() => + EntryMarkdown.Render((await ContentAsync(scenario, content => content.GetAsync(page))).Value, $"/api/v1/entries/{page}.md", id => $"/api/v1/entries/{id}.md"); + + var draft = await MarkdownAsync(); + await PublishAsync(scenario, page, 2); + var published = await MarkdownAsync(); + + Assert.Contains("# A sturdy drill\n", draft, StringComparison.Ordinal); + Assert.Contains("## price\n\n99\n", draft, StringComparison.Ordinal); + Assert.Contains($"## product\n\n[{product}](/api/v1/entries/{product}.md)\n", draft, StringComparison.Ordinal); + Assert.Contains("# Now cheaper\n", published, StringComparison.Ordinal); + Assert.Contains($"url: \"/api/v1/entries/{page}.md\"\n", published, StringComparison.Ordinal); + Assert.Contains("version: 2\n", published, StringComparison.Ordinal); + } + + // A revision of the type waiting for review changes nothing a reader sees: each version is read against the definition it was + // written against. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_published_version_is_read_against_the_definition_it_was_written_against(string provider) + { + await using var scenario = await CreateAsync(provider); + var product = await ProductAsync(scenario, "99"); + var page = await PageAsync(scenario, product, "A sturdy drill"); + await PublishAsync(scenario, page, 1); + var revised = new ContentTypeDefinition([SchemaScenario.Reference("product", "product"), SchemaScenario.Field("body", "richText")]); + Assert.True((await scenario.Schema.ReviseAsync("product-page", 1, revised)).IsSuccess); + + var read = (await ContentAsync(scenario, content => content.GetAsync(page))).Value; + var index = await ContentAsync(scenario, content => content.IndexAsync()); + + Assert.Equal(["product", "price", "headline", "body"], read.Definition.Fields.Select(field => field.Name)); + Assert.Equal("A sturdy drill", Assert.Single(index).Title); } private static async Task CreateAsync(string provider) diff --git a/tests/FluentCMS.Tests/Delivery/EntryMarkdownTests.cs b/tests/FluentCMS.Tests/Delivery/EntryMarkdownTests.cs new file mode 100644 index 000000000..070a797db --- /dev/null +++ b/tests/FluentCMS.Tests/Delivery/EntryMarkdownTests.cs @@ -0,0 +1,265 @@ +using System.Text.Json; +using FluentCMS.Core.Content; +using FluentCMS.Core.Delivery; +using FluentCMS.Core.Schema; + +namespace FluentCMS.Tests.Delivery; + +public sealed class EntryMarkdownTests +{ + private static readonly Guid Page = Guid.Parse("0192f3a1-7c2e-7a4b-9d3e-1f2a3b4c5d6e"); + private static readonly Guid Product = Guid.Parse("0192f3a1-0000-7000-8000-000000000001"); + private static readonly Guid Hero = Guid.Parse("0192f3a1-0000-7000-8000-000000000002"); + + private static readonly ContentTypeDefinition PageType = new( + [ + Field("product", "reference", """{"target": "product"}"""), + Field("price", "reference", """{"target": "product", "field": "price"}"""), + Field("headline", "text"), + Field("body", "richText"), + Field("tags", "list", """{"itemType": "text"}"""), + Field("related", "list", """{"itemType": "reference", "itemSettings": {"target": "product"}}"""), + Field("hero", "media"), + Field("featured", "boolean"), + Field("color", "enum", """{"values": ["red", "blue"]}"""), + Field("launch", "dateTime"), + Field("extra", "json"), + Field("note", "text"), + Field("sections", "list", """{"itemType": "richText"}"""), + Field("maker", "reference", """{"target": "product", "field": null}"""), + ]); + + private static readonly ContentTypeDefinition ProductType = new( + [Field("name", "text"), Field("price", "number"), Field("claim", "richText"), Field("colors", "list", """{"itemType": "text"}""")], + isEntity: true); + + [Fact] + public void A_page_renders_front_matter_its_title_and_each_field_by_its_type() + { + var item = Item(PageType, "product-page", $$""" + { + "product": "{{Product}}", + "price": 99.5, + "headline": "A sturdy\ndrill", + "body": "Built for **daily** use.\n\n- Two speeds\n", + "tags": ["tools", "drills"], + "related": ["{{Product}}"], + "hero": "{{Hero}}", + "featured": true, + "color": "red", + "launch": "2026-10-18T09:00:00+00:00", + "extra": {"warranty": 2}, + "note": null, + "sections": ["First part.\n\nSecond paragraph.", "```\ncode\n```"], + "maker": "{{Product}}" + } + """); + + var markdown = EntryMarkdown.Render(item, $"/api/v1/entries/{Page}.md", id => $"/api/v1/entries/{id}.md"); + + Assert.Equal( + $$""" + --- + id: "{{Page}}" + type: "product-page" + version: 3 + url: "/api/v1/entries/{{Page}}.md" + --- + + # A sturdy drill + + ## product + + [{{Product}}](/api/v1/entries/{{Product}}.md) + + ## price + + 99.5 + + ## body + + > Built for **daily** use. + > + > - Two speeds + + ## tags + + - tools + - drills + + ## related + + - [{{Product}}](/api/v1/entries/{{Product}}.md) + + ## hero + + Media asset `{{Hero}}` + + ## featured + + true + + ## color + + red + + ## launch + + 2026-10-18T09:00:00+00:00 + + ## extra + + ```json + { + "warranty": 2 + } + ``` + + ## sections + + - > First part. + > + > Second paragraph. + - > ``` + > code + > ``` + + ## maker + + [{{Product}}](/api/v1/entries/{{Product}}.md) + + """.Replace("\r\n", "\n", StringComparison.Ordinal), + markdown); + } + + // A value is content, never structure: text that looks like a field, a source, HTML or emphasis reads as written. + [Fact] + public void Text_cannot_pose_as_a_field_a_source_html_or_formatting() + { + var page = Item(PageType, "product-page", """ + {"headline": "Learn C #", "note": "Built tough.\n\n## price\n\n0\n\n costs 2*3*4 & more\n1. first\n- item\n indented\n---"} + """); + + var markdown = EntryMarkdown.Render(page, "/x.md", id => $"/{id}.md"); + + Assert.Contains("# Learn C \\#\n", markdown, StringComparison.Ordinal); + Assert.Contains( + "## note\n\nBuilt tough.\n\n\\## price\n\n0\n\n\\ costs 2\\*3\\*4 \\& more\n1\\. first\n\\- item\nindented\n\\---\n", + markdown, + StringComparison.Ordinal); + } + + // Rich text stays rich, but quoted: a fence or a fake source it holds ends with its field. + [Fact] + public void Rich_text_cannot_reach_the_fields_and_sources_after_it() + { + var claimFirst = new ContentTypeDefinition([Field("name", "text"), Field("claim", "richText"), Field("price", "number")], isEntity: true); + var product = Item(claimFirst, "product", """ + { + "name": {"value": "Drill", "source": {"kind": "person", "reference": "Amir"}, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null}, + "claim": {"value": "Lasts 10 years\n\n**Source:** \n\n```\nopen fence", "source": null, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null}, + "price": {"value": 99, "source": null, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null} + } + """); + + var markdown = EntryMarkdown.Render(product, "/y.md", id => $"/{id}.md"); + + Assert.Contains( + "## claim\n\n> Lasts 10 years\n>\n> **Source:** \n>\n> ```\n> open fence\n\n**Source:** none, so this fact is unsourced\n", + markdown, + StringComparison.Ordinal); + Assert.EndsWith("## price\n\n99\n\n**Source:** none, so this fact is unsourced\n", markdown, StringComparison.Ordinal); + } + + [Fact] + public void An_entity_shows_each_facts_value_with_its_source_the_title_fact_included() + { + var item = Item(ProductType, "product", """ + { + "name": {"value": "Drill", "source": null, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null}, + "price": {"value": 99, "source": {"kind": "url", "reference": "https://example.com/prices_[2026]?a=1&b=2"}, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null}, + "colors": {"value": ["red", "blue"], "source": {"kind": "file", "reference": "exports/catalog_2026.csv"}, "owner": null, "validFrom": null, "validUntil": null, "lastVerified": null} + } + """); + + var markdown = EntryMarkdown.Render(item, "/x.md", id => $"/{id}.md"); + + Assert.Contains("# Drill\n\n## name\n\nDrill\n\n**Source:** none, so this fact is unsourced\n", markdown, StringComparison.Ordinal); + Assert.Contains("## price\n\n99\n\n**Source:** \\(url\\)\n", markdown, StringComparison.Ordinal); + Assert.EndsWith("## colors\n\n- red\n- blue\n\n**Source:** exports/catalog_2026.csv \\(file\\)\n", markdown, StringComparison.Ordinal); + Assert.Equal("Drill", EntryMarkdown.Title(item.Fields, ProductType)); + } + + [Fact] + public void Without_text_the_heading_is_the_type_name_and_the_id_and_unresolved_references_are_listed() + { + var item = Item(PageType, "product-page", $$"""{"product": "{{Product}}", "price": null, "headline": " "}""") with + { + ContentTypeName = "Product\n- [Click](https://evil.example)", + Unresolved = [new UnresolvedReference("/price", Product, "price", UnresolvedReason.NotPublished)], + }; + + var markdown = EntryMarkdown.Render(item, "/x.md", id => $"/{id}.md"); + + Assert.Contains($"# Product - \\[Click\\](https://evil.example) {Page}\n", markdown, StringComparison.Ordinal); + Assert.Null(EntryMarkdown.Title(item.Fields, PageType)); + Assert.DoesNotContain("## headline", markdown, StringComparison.Ordinal); + Assert.EndsWith($"## Unresolved references\n\n- `/price`: price of entry {Product} is not published\n", markdown, StringComparison.Ordinal); + } + + // Link text and plain text: what could open structure is escaped, and words and URLs keep their characters. + [Theory] + [InlineData(@"C:\", @"C:\\")] + [InlineData(@"a\[b]", @"a\\\[b\]")] + [InlineData("", @"\")] + [InlineData("two\nlines", "two lines")] + [InlineData("https://en.wikipedia.org/wiki/Power_drill?a=1&b=2#history", "https://en.wikipedia.org/wiki/Power_drill?a=1&b=2#history")] + [InlineData("_emphasis_ and snake_case", @"\_emphasis\_ and snake_case")] + [InlineData("C# and F#", "C# and F#")] + [InlineData("Title ##", @"Title \##")] + [InlineData("© and © and AT&T", @"\© and \© and AT&T")] + [InlineData("$$", @"\$$")] + [InlineData("^^^ figure", @"\^^^ figure")] + public void Inline_text_escapes_only_what_could_open_structure(string text, string expected) + { + Assert.Equal(expected, EntryMarkdown.InlineText(text)); + } + + // CommonMark lets an item start with one blank line at most, so leading blank lines would push the text out of its bullet. + [Fact] + public void A_list_item_that_starts_with_blank_lines_stays_in_its_bullet() + { + var item = Item(PageType, "product-page", """{"tags": ["one", "\n\nsecond item", "third"]}"""); + + var markdown = EntryMarkdown.Render(item, "/x.md", id => $"/{id}.md"); + + Assert.Contains("## tags\n\n- one\n- second item\n- third\n", markdown, StringComparison.Ordinal); + } + + [Fact] + public void A_backtick_in_json_cannot_close_its_fence() + { + var item = Item(PageType, "product-page", """{"extra": {"snippet": "```` four"}}"""); + + var markdown = EntryMarkdown.Render(item, "/x.md", id => $"/{id}.md"); + + Assert.Contains("```json\n{\n \"snippet\": \"\\u0060\\u0060\\u0060\\u0060 four\"\n}\n```\n", markdown, StringComparison.Ordinal); + } + + private static PublishedItem Item(ContentTypeDefinition definition, string type, string fields) + { + return new PublishedItem( + Page, + type, + type == "product" ? "Product" : "Product page", + definition, + 3, + JsonSerializer.Deserialize(fields), + []); + } + + private static FieldDefinition Field(string name, string type, string settings = "{}") + { + return new FieldDefinition(name, type, settings: JsonSerializer.Deserialize(settings)); + } +}