Add the search index on PostgreSQL and SQLite and the search tool

CONTRIBUTING.md

@@ -138,6 +138,8 @@ An LLM-judged type asks a model only through `IModelGateway`, which it reads fro
 
 A check is a versioned definition under a slug: a check type and its settings, a description of what it enforces, a severity (block or warn), an optional scope of content types and fields, and test examples, at least one it must pass and one it must fail. `ICheckDirectory` activates a version only after running its examples, and names each example that does not get its verdict (CHK-02). A revision leaves the active version in place until the new one is activated, and only a person activates or deactivates a check. `ICheckRunner` runs the active checks that apply to a piece of content and gives each one's status: pass, warn for a failing check of warn severity, fail, or needs review. Agents run it on a draft with the `check_draft` tool before they propose it (CHK-04).
 
+Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension<ISearchIndex, T>(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads the matches a page at a time, `Search__PageSize` (1,000 by default), and drops what the scope, deletions and filters leave out, until enough are left, the index has no more, or it has read `Search__MaxMatches` (10,000). A result that stopped there says it is not complete. Each hit has the state the entry tools show, in review included. A string that is a GUID, such as a reference, is not indexed.
+
 ## Code style
 
 `.editorconfig` is the source of truth: UTF-8, LF line endings, four spaces in C#, two in JSON and YAML, file-scoped namespaces. Nullable reference types and implicit usings are enabled project-wide. Prefer clear code over clever code. Comments explain why, not what. Do not add XML documentation comments unless an issue asks for them.

src/FluentCMS.Core/Builtin/Tools/SearchTools.cs

@@ -0,0 +1,92 @@
+using System.Text.Json.Nodes;
+using FluentCMS.Core.Content;
+using FluentCMS.Core.Results;
+using FluentCMS.Core.Search;
+using FluentCMS.Core.Tools;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FluentCMS.Core.Builtin.Tools;
+
+// Finding entries and entities by their words (SRC-01, MCP-01), within the types the agent may read.
+public sealed class SearchTools : IToolProvider
+{
+    private static readonly string StateNames = string.Join(", ", Enum.GetValues<EntryState>().Select(state => ToolJson.Name(state)));
+
+    public IReadOnlyList<AgentTool> Tools { get; } =
+    [
+        new AgentTool(
+            "search_entries",
+            "Search entries and entities",
+            "Finds entries and entities whose latest version has every word of the query, each matched as a prefix, best first: "
+            + "'4411' finds a SKU and '441' does too. An entry that references a match is found as well, with the match in 'via', "
+            + "such as the page that shows a matching entity. Narrow the search with types, states or published_only; read a hit "
+            + "with get_entry, or an entity's facts with get_entity_facts.",
+            ToolSchemas.Object(
+                ToolSchemas.String("query", "The words to find, such as a name, a SKU or part of a sentence.", "4411"),
+                ToolSchemas.Strings("types", "Content type slugs to search in; leave out for every type you may read.", "product", required: false),
+                ToolSchemas.Strings("states", $"Entry states to keep: {StateNames}.", "published", required: false),
+                ToolSchemas.Flag("published_only", "Only entries that have a published version."),
+                ToolSchemas.PositiveInteger("limit", $"At most this many hits, up to {SearchQuery.MaxLimit}; {SearchQuery.DefaultLimit} if left out.", 20, required: false)),
+            readOnly: true,
+            SearchAsync),
+    ];
+
+    private static async Task<ToolResult> SearchAsync(ToolInvocation call, CancellationToken cancellationToken)
+    {
+        var arguments = new ToolArgumentsReader(call.Arguments);
+        var text = arguments.RequiredString("query", "4411");
+        var types = arguments.OptionalStrings("types", "product");
+        var states = arguments.OptionalStrings("states", "published");
+        var publishedOnly = arguments.Flag("published_only");
+        var limit = arguments.OptionalPositiveInteger("limit") ?? SearchQuery.DefaultLimit;
+        var errors = new List<Error>(arguments.Errors());
+
+        var parsedStates = new List<EntryState>();
+        foreach (var name in states ?? [])
+        {
+            var state = Enum.GetValues<EntryState>().Where(candidate => ToolJson.Name(candidate) == name).ToList();
+            if (state.Count == 1)
+            {
+                parsedStates.Add(state[0]);
+            }
+            else
+            {
+                errors.Add(new Error(ToolErrorCodes.ArgumentInvalid, $"'{name}' is not an entry state; use {StateNames}."));
+            }
+        }
+
+        if (errors.Count > 0)
+        {
+            return ToolResult.Failure(errors);
+        }
+
+        var results = await call.Services.GetRequiredService<ISearch>().SearchAsync(
+            new SearchQuery(text!)
+            {
+                ContentTypes = types,
+                States = states is null ? null : parsedStates,
+                PublishedOnly = publishedOnly,
+                Limit = limit,
+            },
+            cancellationToken);
+
+        return ToolResult.From(results, found => new JsonObject
+        {
+            ["hits"] = new JsonArray([.. found.Hits.Select(hit => new JsonObject
+            {
+                ["id"] = hit.EntryId.ToString(),
+                ["contentType"] = hit.ContentType,
+                ["state"] = ToolJson.Name(hit.State),
+                ["latestVersion"] = hit.LatestVersion,
+                ["publishedVersion"] = hit.PublishedVersion,
+                ["via"] = hit.Via?.ToString(),
+            })]),
+            ["complete"] = found.Complete,
+            ["next"] = !found.Complete
+                ? "The search stopped after reading the most matches it reads, so hits may be missing. Add a word to narrow it."
+                : found.Hits.Count == 0
+                    ? "Nothing matched. Search for fewer or shorter words, or leave out a filter."
+                    : "Read a hit with get_entry, or an entity's facts with get_entity_facts. A hit with 'via' references that match.",
+        });
+    }
+}

src/FluentCMS.Core/Extensions/ExtensionServiceCollectionExtensions.cs

@@ -6,14 +6,21 @@ namespace FluentCMS.Core.Extensions;
 
 public static class ExtensionServiceCollectionExtensions
 {
-    public static IServiceCollection AddFluentCmsExtension<TContract, TImplementation>(this IServiceCollection services)
+    public static IServiceCollection AddFluentCmsExtension<TContract, TImplementation>(
+        this IServiceCollection services,
+        ServiceLifetime lifetime = ServiceLifetime.Singleton)
         where TContract : class
         where TImplementation : class, TContract
     {
-        return services.AddFluentCmsExtension(typeof(TContract), typeof(TImplementation));
+        return services.AddFluentCmsExtension(typeof(TContract), typeof(TImplementation), lifetime);
     }
 
-    public static IServiceCollection AddFluentCmsExtension(this IServiceCollection services, Type contract, Type implementation)
+    // Singleton unless the implementation needs a scope, such as one that reads the database through the request's context.
+    public static IServiceCollection AddFluentCmsExtension(
+        this IServiceCollection services,
+        Type contract,
+        Type implementation,
+        ServiceLifetime lifetime = ServiceLifetime.Singleton)
     {
         if (!IsExtensionPoint(contract))
         {
@@ -36,7 +43,7 @@ public static IServiceCollection AddFluentCmsExtension(this IServiceCollection s
             return services;
         }
 
-        services.AddSingleton(contract, implementation);
+        services.Add(new ServiceDescriptor(contract, implementation, lifetime));
         services.AddSingleton(new ExtensionDescriptor(contract, implementation));
 
         return services;

src/FluentCMS.Core/Search/ISearch.cs

@@ -0,0 +1,49 @@
+using FluentCMS.Core.Content;
+using FluentCMS.Core.Results;
+
+namespace FluentCMS.Core.Search;
+
+// Finding entries and entities by text with filters (SRC-01), for the search tool and the desk. Only the types the caller may read
+// are searched, so search never shows more than the read tools would (MCP-05).
+public interface ISearch
+{
+    Task<Result<SearchResults>> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default);
+}
+
+public sealed record SearchQuery(string Text)
+{
+    public const int DefaultLimit = 20;
+    public const int MaxLimit = 100;
+
+    // Content type slugs; null for every type.
+    public IReadOnlyList<string>? ContentTypes { get; init; }
+
+    // Null for every state.
+    public IReadOnlyList<EntryState>? States { get; init; }
+
+    public bool PublishedOnly { get; init; }
+
+    public int Limit { get; init; } = DefaultLimit;
+}
+
+// Complete is false when the search stopped at the most matches it reads, so better-ranked matches outside the filters or the
+// scope may have crowded out some that would pass them.
+public sealed record SearchResults(IReadOnlyList<SearchHit> Hits, bool Complete = true);
+
+// A direct match has no Via; an entry that references a match names it in Via, such as the page that shows a matching entity.
+public sealed record SearchHit(
+    Guid EntryId,
+    string ContentType,
+    EntryState State,
+    int LatestVersion,
+    int? PublishedVersion,
+    double Rank,
+    Guid? Via);
+
+public static class SearchErrorCodes
+{
+    public const string QueryEmpty = "search_query_empty";
+    public const string QueryTooLong = "search_query_too_long";
+    public const string LimitInvalid = "search_limit_invalid";
+    public const string FilterEmpty = "search_filter_empty";
+}

src/FluentCMS.Core/Search/ISearchIndex.cs

@@ -0,0 +1,21 @@
+using FluentCMS.Core.Extensions;
+
+namespace FluentCMS.Core.Search;
+
+// The full-text index over entries and entities (SRC-01, ADR-0010): PostgreSQL full-text search or SQLite FTS5 in the application
+// database, or an external engine a plugin adds later. IndexAsync is called inside the unit of work of the write it reflects, so an
+// entry is found the moment its write commits and never when the write rolled back.
+[ExtensionPoint]
+public interface ISearchIndex
+{
+    // Replaces what the index holds for the entry.
+    Task IndexAsync(SearchDocument document, CancellationToken cancellationToken = default);
+
+    // Entries whose text has every term, each matched as a prefix, best first and in the same order on every call, so a caller pages
+    // through them by skipping those it has read. Terms are words of letters and digits (SearchTerms).
+    Task<IReadOnlyList<SearchMatch>> MatchAsync(IReadOnlyList<string> terms, int skip, int limit, CancellationToken cancellationToken = default);
+}
+
+public sealed record SearchDocument(Guid EntryId, string Text);
+
+public sealed record SearchMatch(Guid EntryId, double Rank);

src/FluentCMS.Core/Search/SearchText.cs

@@ -0,0 +1,82 @@
+using System.Globalization;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace FluentCMS.Core.Search;
+
+// What an entry is found by: every string and number in its fields, a fact's value and source included, in field order, as the
+// lowercase words SearchTerms splits a query into. A string that is a GUID, such as a reference or a media asset's id, is left out:
+// its parts would match queries such as "01". Indexing words rather than raw text keeps both engines' tokenizers out of it, so
+// "19.99" or "A-4411" match the same way on PostgreSQL and SQLite. The migrations that index existing entries do the same.
+public static class SearchText
+{
+    // Well inside PostgreSQL's 1 MB limit on a tsvector, so even a very long entry is stored; words past it are not searchable.
+    public const int MaxLength = 100_000;
+
+    public static string From(JsonElement fields)
+    {
+        var text = new StringBuilder();
+        Append(fields, text);
+
+        var words = new StringBuilder();
+        foreach (var word in SearchTerms.Words(text.ToString()))
+        {
+            if (words.Length + word.Length + 1 > MaxLength)
+            {
+                break;
+            }
+
+            words.Append(words.Length == 0 ? string.Empty : " ").Append(word);
+        }
+
+        return words.ToString();
+    }
+
+    private static void Append(JsonElement value, StringBuilder text)
+    {
+        switch (value.ValueKind)
+        {
+            case JsonValueKind.Object:
+                foreach (var property in value.EnumerateObject())
+                {
+                    Append(property.Value, text);
+                }
+
+                break;
+            case JsonValueKind.Array:
+                foreach (var item in value.EnumerateArray())
+                {
+                    Append(item, text);
+                }
+
+                break;
+            case JsonValueKind.String when !Guid.TryParseExact(value.GetString(), "D", out _):
+                text.Append(value.GetString()).Append(' ');
+                break;
+            case JsonValueKind.Number:
+                text.Append(value.GetRawText()).Append(' ');
+                break;
+        }
+    }
+}
+
+// A query split into the words both indexes match: runs of letters and digits, lowercased, as the indexes' tokenizers split text.
+public static partial class SearchTerms
+{
+    public const int MaxTerms = 10;
+    public const int MaxTermLength = 64;
+
+    public static IReadOnlyList<string> From(string? query)
+    {
+        return query is null ? [] : [.. Words(query).Distinct(StringComparer.Ordinal)];
+    }
+
+    internal static IEnumerable<string> Words(string text)
+    {
+        return Word().Matches(text).Select(match => match.Value.ToLower(CultureInfo.InvariantCulture));
+    }
+
+    [GeneratedRegex(@"[\p{L}\p{N}]+")]
+    private static partial Regex Word();
+}

src/FluentCMS.Core/Tools/ToolArgumentsReader.cs

@@ -94,6 +94,22 @@ public ToolArgumentsReader(JsonElement arguments)
         return null;
     }
 
+    public bool Flag(string name)
+    {
+        if (Read(name) is not { } value)
+        {
+            return false;
+        }
+
+        if (value.ValueKind is JsonValueKind.True or JsonValueKind.False)
+        {
+            return value.GetBoolean();
+        }
+
+        Invalid(name, "must be true or false");
+        return false;
+    }
+
     public Guid? OptionalGuid(string name)
     {
         return Read(name) is null ? null : GuidOf(name);

src/FluentCMS.Core/Tools/ToolSchemas.cs

@@ -60,6 +60,11 @@ public static ToolArgument Strings(string name, string description, string examp
             });
     }
 
+    public static ToolArgument Flag(string name, string description)
+    {
+        return new ToolArgument(name, description, Required: false, new JsonObject { ["type"] = "boolean", ["examples"] = new JsonArray(true) });
+    }
+
     // An argument that is itself a JSON object, with its own schema and one example that satisfies it.
     public static ToolArgument Json(string name, string description, JsonObject schema, JsonNode example, bool required = true)
     {

src/FluentCMS.Infrastructure/Content/EntryDirectory.cs

@@ -7,6 +7,7 @@
 using FluentCMS.Core.Persistence;
 using FluentCMS.Core.Results;
 using FluentCMS.Core.Schema;
+using FluentCMS.Core.Search;
 using FluentCMS.Core.Sites;
 using FluentCMS.Infrastructure.Entities;
 using FluentCMS.Infrastructure.Persistence;
@@ -26,7 +27,8 @@ internal sealed class EntryDirectory(
     FieldTypeRegistry fieldTypes,
     TimeProvider clock,
     IOptions<ContentOptions> options,
-    IAgentPermissions permissions) : IEntryDirectory
+    IAgentPermissions permissions,
+    ISearchIndex searchIndex) : IEntryDirectory
 {
     // Until multi-site (MST-01), every entry belongs to the one site (SCH-07).
     private static readonly Guid SiteId = Site.Default.Id;
@@ -176,6 +178,7 @@ public async Task<Result<Entry>> CreateAsync(NewEntry entry, CancellationToken c
         context.EntryVersions.Add(VersionRow(stored.Latest));
         context.EntryReferences.AddRange(ReferenceRows(stored.Latest, references));
         context.EntityFacts.AddRange(FactRows(stored.Latest, type.Value.Definition));
+        await searchIndex.IndexAsync(new SearchDocument(stored.Id, SearchText.From(stored.Latest.Fields)), cancellationToken);
         var subject = Subject(type.Value.Definition);
         await auditLog.AppendAsync(new NewAuditEvent($"{subject}.created") { SubjectType = subject, SubjectId = stored.Id }, cancellationToken);
 
@@ -245,6 +248,7 @@ public async Task<Result<Entry>> UpdateAsync(Guid id, int baseVersion, JsonEleme
         context.EntryVersions.Add(VersionRow(entry.Latest));
         context.EntryReferences.AddRange(ReferenceRows(entry.Latest, references));
         context.EntityFacts.AddRange(FactRows(entry.Latest, type.Value.Definition));
+        await searchIndex.IndexAsync(new SearchDocument(id, SearchText.From(entry.Latest.Fields)), cancellationToken);
         var subject = Subject(type.Value.Definition);
         await auditLog.AppendAsync(
             new NewAuditEvent($"{subject}.updated") { SubjectType = subject, SubjectId = id, PreviousVersionId = baseRow.Id },