From d15efd725dbcaed5a0c8558f47390ac274f95145 Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Sun, 18 Oct 2026 23:38:36 -0400 Subject: [PATCH 1/5] Add the search index on PostgreSQL and SQLite FTS5 and the search tool --- CONTRIBUTING.md | 2 + .../Builtin/Tools/SearchTools.cs | 86 ++ .../ExtensionServiceCollectionExtensions.cs | 15 +- src/FluentCMS.Core/Search/ISearch.cs | 46 + src/FluentCMS.Core/Search/ISearchIndex.cs | 20 + src/FluentCMS.Core/Search/SearchText.cs | 66 + .../Tools/ToolArgumentsReader.cs | 16 + src/FluentCMS.Core/Tools/ToolSchemas.cs | 5 + .../Content/EntryDirectory.cs | 6 +- ...frastructureServiceCollectionExtensions.cs | 21 +- ...20261019033140_SearchDocuments.Designer.cs | 1189 +++++++++++++++++ .../20261019033140_SearchDocuments.cs | 44 + ...20261019033130_SearchDocuments.Designer.cs | 1179 ++++++++++++++++ .../Sqlite/20261019033130_SearchDocuments.cs | 39 + .../Search/EntrySearch.cs | 135 ++ .../Search/SearchIndexes.cs | 79 ++ .../Mcp/BuiltInToolsTests.cs | 2 +- .../Mcp/McpAgents.cs | 3 +- .../Mcp/SearchToolsTests.cs | 48 + .../Search/SearchTests.cs | 149 +++ .../FluentCMS.Tests/Search/SearchTextTests.cs | 34 + .../FluentCMS.Tests/Tools/McpCoverageTests.cs | 2 +- .../Tools/ToolProviderConformanceTests.cs | 5 + 23 files changed, 3182 insertions(+), 9 deletions(-) create mode 100644 src/FluentCMS.Core/Builtin/Tools/SearchTools.cs create mode 100644 src/FluentCMS.Core/Search/ISearch.cs create mode 100644 src/FluentCMS.Core/Search/ISearchIndex.cs create mode 100644 src/FluentCMS.Core/Search/SearchText.cs create mode 100644 src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs create mode 100644 src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs create mode 100644 src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs create mode 100644 src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs create mode 100644 src/FluentCMS.Infrastructure/Search/EntrySearch.cs create mode 100644 src/FluentCMS.Infrastructure/Search/SearchIndexes.cs create mode 100644 tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs create mode 100644 tests/FluentCMS.IntegrationTests/Search/SearchTests.cs create mode 100644 tests/FluentCMS.Tests/Search/SearchTextTests.cs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dc66c45e..aa959432f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -135,6 +135,8 @@ An LLM-judged type asks a model only through `IModelGateway`, which it reads fro A check is a versioned definition under a slug: a check type and its settings, a description of what it enforces, a severity (block or warn), an optional scope of content types and fields, and test examples, at least one it must pass and one it must fail. `ICheckDirectory` activates a version only after running its examples, and names each example that does not get its verdict (CHK-02). A revision leaves the active version in place until the new one is activated, and only a person activates or deactivates a check. `ICheckRunner` runs the active checks that apply to a piece of content and gives each one's status: pass, warn for a failing check of warn severity, fail, or needs review. Agents run it on a draft with the `check_draft` tool before they propose it (CHK-04). +Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. + ## Code style `.editorconfig` is the source of truth: UTF-8, LF line endings, four spaces in C#, two in JSON and YAML, file-scoped namespaces. Nullable reference types and implicit usings are enabled project-wide. Prefer clear code over clever code. Comments explain why, not what. Do not add XML documentation comments unless an issue asks for them. diff --git a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs new file mode 100644 index 000000000..e54fdd21d --- /dev/null +++ b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs @@ -0,0 +1,86 @@ +using System.Text.Json.Nodes; +using FluentCMS.Core.Content; +using FluentCMS.Core.Results; +using FluentCMS.Core.Search; +using FluentCMS.Core.Tools; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Core.Builtin.Tools; + +// Finding entries and entities by their words (SRC-01, MCP-01), within the types the agent may read. +public sealed class SearchTools : IToolProvider +{ + private static readonly string StateNames = string.Join(", ", Enum.GetValues().Select(state => ToolJson.Name(state))); + + public IReadOnlyList Tools { get; } = + [ + new AgentTool( + "search_entries", + "Search entries and entities", + "Finds entries and entities whose latest version has every word of the query, each matched as a prefix, best first: " + + "'4411' finds a SKU and '441' does too. An entry that references a match is found as well, with the match in 'via', " + + "such as the page that shows a matching entity. Narrow the search with types, states or published_only; read a hit " + + "with get_entry, or an entity's facts with get_entity_facts.", + ToolSchemas.Object( + ToolSchemas.String("query", "The words to find, such as a name, a SKU or part of a sentence.", "4411"), + ToolSchemas.Strings("types", "Content type slugs to search in; leave out for every type you may read.", "product", required: false), + ToolSchemas.Strings("states", $"Entry states to keep: {StateNames}.", "published", required: false), + ToolSchemas.Flag("published_only", "Only entries that have a published version."), + ToolSchemas.PositiveInteger("limit", $"At most this many hits, up to {SearchQuery.MaxLimit}; {SearchQuery.DefaultLimit} if left out.", 20, required: false)), + readOnly: true, + SearchAsync), + ]; + + private static async Task SearchAsync(ToolInvocation call, CancellationToken cancellationToken) + { + var arguments = new ToolArgumentsReader(call.Arguments); + var text = arguments.RequiredString("query", "4411"); + var types = arguments.OptionalStrings("types", "product"); + var states = arguments.OptionalStrings("states", "published"); + var publishedOnly = arguments.Flag("published_only"); + var limit = arguments.OptionalPositiveInteger("limit") ?? SearchQuery.DefaultLimit; + var errors = new List(arguments.Errors()); + + var parsedStates = new List(); + foreach (var name in states ?? []) + { + var state = Enum.GetValues().Where(candidate => ToolJson.Name(candidate) == name).ToList(); + if (state.Count == 1) + { + parsedStates.Add(state[0]); + } + else + { + errors.Add(new Error(ToolErrorCodes.ArgumentInvalid, $"'{name}' is not an entry state; use {StateNames}.")); + } + } + + if (errors.Count > 0) + { + return ToolResult.Failure(errors); + } + + var results = await call.Services.GetRequiredService().SearchAsync( + new SearchQuery(text!) + { + ContentTypes = types, + States = states is null ? null : parsedStates, + PublishedOnly = publishedOnly, + Limit = limit, + }, + cancellationToken); + + return ToolResult.From(results, found => new JsonObject + { + ["hits"] = new JsonArray([.. found.Hits.Select(hit => new JsonObject + { + ["id"] = hit.EntryId.ToString(), + ["contentType"] = hit.ContentType, + ["state"] = ToolJson.Name(hit.State), + ["latestVersion"] = hit.LatestVersion, + ["publishedVersion"] = hit.PublishedVersion, + ["via"] = hit.Via?.ToString(), + })]), + }); + } +} diff --git a/src/FluentCMS.Core/Extensions/ExtensionServiceCollectionExtensions.cs b/src/FluentCMS.Core/Extensions/ExtensionServiceCollectionExtensions.cs index da4396fe3..7213562ae 100644 --- a/src/FluentCMS.Core/Extensions/ExtensionServiceCollectionExtensions.cs +++ b/src/FluentCMS.Core/Extensions/ExtensionServiceCollectionExtensions.cs @@ -6,14 +6,21 @@ namespace FluentCMS.Core.Extensions; public static class ExtensionServiceCollectionExtensions { - public static IServiceCollection AddFluentCmsExtension(this IServiceCollection services) + public static IServiceCollection AddFluentCmsExtension( + this IServiceCollection services, + ServiceLifetime lifetime = ServiceLifetime.Singleton) where TContract : class where TImplementation : class, TContract { - return services.AddFluentCmsExtension(typeof(TContract), typeof(TImplementation)); + return services.AddFluentCmsExtension(typeof(TContract), typeof(TImplementation), lifetime); } - public static IServiceCollection AddFluentCmsExtension(this IServiceCollection services, Type contract, Type implementation) + // Singleton unless the implementation needs a scope, such as one that reads the database through the request's context. + public static IServiceCollection AddFluentCmsExtension( + this IServiceCollection services, + Type contract, + Type implementation, + ServiceLifetime lifetime = ServiceLifetime.Singleton) { if (!IsExtensionPoint(contract)) { @@ -36,7 +43,7 @@ public static IServiceCollection AddFluentCmsExtension(this IServiceCollection s return services; } - services.AddSingleton(contract, implementation); + services.Add(new ServiceDescriptor(contract, implementation, lifetime)); services.AddSingleton(new ExtensionDescriptor(contract, implementation)); return services; diff --git a/src/FluentCMS.Core/Search/ISearch.cs b/src/FluentCMS.Core/Search/ISearch.cs new file mode 100644 index 000000000..2abc1bc6d --- /dev/null +++ b/src/FluentCMS.Core/Search/ISearch.cs @@ -0,0 +1,46 @@ +using FluentCMS.Core.Content; +using FluentCMS.Core.Results; + +namespace FluentCMS.Core.Search; + +// Finding entries and entities by text with filters (SRC-01), for the search tool and the desk. Only the types the caller may read +// are searched, so search never shows more than the read tools would (MCP-05). +public interface ISearch +{ + Task> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default); +} + +public sealed record SearchQuery(string Text) +{ + public const int DefaultLimit = 20; + public const int MaxLimit = 100; + + // Content type slugs; null for every type. + public IReadOnlyList? ContentTypes { get; init; } + + // Null for every state. + public IReadOnlyList? States { get; init; } + + public bool PublishedOnly { get; init; } + + public int Limit { get; init; } = DefaultLimit; +} + +public sealed record SearchResults(IReadOnlyList Hits); + +// A direct match has no Via; an entry that references a match names it in Via, such as the page that shows a matching entity. +public sealed record SearchHit( + Guid EntryId, + string ContentType, + EntryState State, + int LatestVersion, + int? PublishedVersion, + double Rank, + Guid? Via); + +public static class SearchErrorCodes +{ + public const string QueryEmpty = "search_query_empty"; + public const string QueryTooLong = "search_query_too_long"; + public const string LimitInvalid = "search_limit_invalid"; +} diff --git a/src/FluentCMS.Core/Search/ISearchIndex.cs b/src/FluentCMS.Core/Search/ISearchIndex.cs new file mode 100644 index 000000000..211731c15 --- /dev/null +++ b/src/FluentCMS.Core/Search/ISearchIndex.cs @@ -0,0 +1,20 @@ +using FluentCMS.Core.Extensions; + +namespace FluentCMS.Core.Search; + +// The full-text index over entries and entities (SRC-01, ADR-0010): PostgreSQL full-text search or SQLite FTS5 in the application +// database, or an external engine a plugin adds later. IndexAsync is called inside the unit of work of the write it reflects, so an +// entry is found the moment its write commits and never when the write rolled back. +[ExtensionPoint] +public interface ISearchIndex +{ + // Replaces what the index holds for the entry. + Task IndexAsync(SearchDocument document, CancellationToken cancellationToken = default); + + // Entries whose text has every term, each matched as a prefix, best first. Terms are words of letters and digits (SearchTerms). + Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default); +} + +public sealed record SearchDocument(Guid EntryId, string Text); + +public sealed record SearchMatch(Guid EntryId, double Rank); diff --git a/src/FluentCMS.Core/Search/SearchText.cs b/src/FluentCMS.Core/Search/SearchText.cs new file mode 100644 index 000000000..bf56afb69 --- /dev/null +++ b/src/FluentCMS.Core/Search/SearchText.cs @@ -0,0 +1,66 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace FluentCMS.Core.Search; + +// What an entry is found by: every string and number in its fields, a fact's value and source included, in field order, as the +// lowercase words SearchTerms splits a query into. Indexing words rather than raw text keeps both engines' tokenizers out of it, so +// "19.99" or "A-4411" match the same way on PostgreSQL and SQLite. The migrations that index existing entries do the same. +public static class SearchText +{ + public static string From(JsonElement fields) + { + var text = new StringBuilder(); + Append(fields, text); + return string.Join(' ', SearchTerms.Words(text.ToString())); + } + + private static void Append(JsonElement value, StringBuilder text) + { + switch (value.ValueKind) + { + case JsonValueKind.Object: + foreach (var property in value.EnumerateObject()) + { + Append(property.Value, text); + } + + break; + case JsonValueKind.Array: + foreach (var item in value.EnumerateArray()) + { + Append(item, text); + } + + break; + case JsonValueKind.String: + text.Append(value.GetString()).Append(' '); + break; + case JsonValueKind.Number: + text.Append(value.GetRawText()).Append(' '); + break; + } + } +} + +// A query split into the words both indexes match: runs of letters and digits, lowercased, as the indexes' tokenizers split text. +public static partial class SearchTerms +{ + public const int MaxTerms = 10; + public const int MaxTermLength = 64; + + public static IReadOnlyList From(string? query) + { + return query is null ? [] : [.. Words(query).Distinct(StringComparer.Ordinal)]; + } + + internal static IEnumerable Words(string text) + { + return Word().Matches(text).Select(match => match.Value.ToLower(CultureInfo.InvariantCulture)); + } + + [GeneratedRegex(@"[\p{L}\p{N}]+")] + private static partial Regex Word(); +} diff --git a/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs b/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs index e544c706f..fdc948a7c 100644 --- a/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs +++ b/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs @@ -94,6 +94,22 @@ public ToolArgumentsReader(JsonElement arguments) return null; } + public bool Flag(string name) + { + if (Read(name) is not { } value) + { + return false; + } + + if (value.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + return value.GetBoolean(); + } + + Invalid(name, "must be true or false"); + return false; + } + public Guid? OptionalGuid(string name) { return Read(name) is null ? null : GuidOf(name); diff --git a/src/FluentCMS.Core/Tools/ToolSchemas.cs b/src/FluentCMS.Core/Tools/ToolSchemas.cs index 648e5f76c..aece4168a 100644 --- a/src/FluentCMS.Core/Tools/ToolSchemas.cs +++ b/src/FluentCMS.Core/Tools/ToolSchemas.cs @@ -60,6 +60,11 @@ public static ToolArgument Strings(string name, string description, string examp }); } + public static ToolArgument Flag(string name, string description) + { + return new ToolArgument(name, description, Required: false, new JsonObject { ["type"] = "boolean", ["examples"] = new JsonArray(true) }); + } + // An argument that is itself a JSON object, with its own schema and one example that satisfies it. public static ToolArgument Json(string name, string description, JsonObject schema, JsonNode example, bool required = true) { diff --git a/src/FluentCMS.Infrastructure/Content/EntryDirectory.cs b/src/FluentCMS.Infrastructure/Content/EntryDirectory.cs index ea100b81d..7f491f6f2 100644 --- a/src/FluentCMS.Infrastructure/Content/EntryDirectory.cs +++ b/src/FluentCMS.Infrastructure/Content/EntryDirectory.cs @@ -7,6 +7,7 @@ using FluentCMS.Core.Persistence; using FluentCMS.Core.Results; using FluentCMS.Core.Schema; +using FluentCMS.Core.Search; using FluentCMS.Core.Sites; using FluentCMS.Infrastructure.Entities; using FluentCMS.Infrastructure.Persistence; @@ -26,7 +27,8 @@ internal sealed class EntryDirectory( FieldTypeRegistry fieldTypes, TimeProvider clock, IOptions options, - IAgentPermissions permissions) : IEntryDirectory + IAgentPermissions permissions, + ISearchIndex searchIndex) : IEntryDirectory { // Until multi-site (MST-01), every entry belongs to the one site (SCH-07). private static readonly Guid SiteId = Site.Default.Id; @@ -176,6 +178,7 @@ public async Task> CreateAsync(NewEntry entry, CancellationToken c context.EntryVersions.Add(VersionRow(stored.Latest)); context.EntryReferences.AddRange(ReferenceRows(stored.Latest, references)); context.EntityFacts.AddRange(FactRows(stored.Latest, type.Value.Definition)); + await searchIndex.IndexAsync(new SearchDocument(stored.Id, SearchText.From(stored.Latest.Fields)), cancellationToken); var subject = Subject(type.Value.Definition); await auditLog.AppendAsync(new NewAuditEvent($"{subject}.created") { SubjectType = subject, SubjectId = stored.Id }, cancellationToken); @@ -245,6 +248,7 @@ public async Task> UpdateAsync(Guid id, int baseVersion, JsonEleme context.EntryVersions.Add(VersionRow(entry.Latest)); context.EntryReferences.AddRange(ReferenceRows(entry.Latest, references)); context.EntityFacts.AddRange(FactRows(entry.Latest, type.Value.Definition)); + await searchIndex.IndexAsync(new SearchDocument(id, SearchText.From(entry.Latest.Fields)), cancellationToken); var subject = Subject(type.Value.Definition); await auditLog.AppendAsync( new NewAuditEvent($"{subject}.updated") { SubjectType = subject, SubjectId = id, PreviousVersionId = baseRow.Id }, diff --git a/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs b/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs index d9dd9264e..9c0da8fab 100644 --- a/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs +++ b/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs @@ -11,6 +11,7 @@ using FluentCMS.Core.Models; using FluentCMS.Core.Persistence; using FluentCMS.Core.Schema; +using FluentCMS.Core.Search; using FluentCMS.Infrastructure.Agents; using FluentCMS.Infrastructure.Audit; using FluentCMS.Infrastructure.ChangeSets; @@ -23,6 +24,7 @@ using FluentCMS.Infrastructure.Models; using FluentCMS.Infrastructure.Persistence; using FluentCMS.Infrastructure.Schema; +using FluentCMS.Infrastructure.Search; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -38,7 +40,9 @@ public static IServiceCollection AddFluentCmsInfrastructure(this IServiceCollect services.Configure(configuration.GetSection(DatabaseOptions.SectionName)); services.TryAddSingleton(TimeProvider.System); - AddDatabase(services, configuration.GetSection(DatabaseOptions.SectionName).Get() ?? new DatabaseOptions()); + var database = configuration.GetSection(DatabaseOptions.SectionName).Get() ?? new DatabaseOptions(); + AddDatabase(services, database); + AddSearch(services, database.ResolveProvider()); AddIdentity(services); AddAudit(services); AddSchema(services); @@ -72,6 +76,21 @@ private static void AddDatabase(IServiceCollection services, DatabaseOptions dat } } + // The index of the configured provider, scoped because it writes through the request's context (ADR-0010). + private static void AddSearch(IServiceCollection services, string provider) + { + if (provider == DatabaseProviders.PostgreSql) + { + services.AddFluentCmsExtension(ServiceLifetime.Scoped); + } + else + { + services.AddFluentCmsExtension(ServiceLifetime.Scoped); + } + + services.AddScoped(); + } + // The log reads ICurrentInitiator, which each host registers for its own kind of caller. private static void AddAudit(IServiceCollection services) { diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs new file mode 100644 index 000000000..649917357 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs @@ -0,0 +1,1189 @@ +// +using System; +using System.Collections.Generic; +using FluentCMS.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FluentCMS.Infrastructure.Persistence.Migrations.PostgreSql +{ + [DbContext(typeof(PostgreSqlFluentCmsDbContext))] + [Migration("20261019033140_SearchDocuments")] + partial class SearchDocuments + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.AgentCredentialRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CallPeriod") + .HasColumnType("interval"); + + b.Property("CallsPerPeriod") + .HasColumnType("integer"); + + b.Property("ChangeSetsPerDay") + .HasColumnType("integer"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.PrimitiveCollection>("Fields") + .HasColumnType("text[]"); + + b.Property("IssuedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ItemsPerChangeSet") + .HasColumnType("integer"); + + b.Property("KeyHash") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("LlmSpendPerPeriod") + .HasColumnType("numeric"); + + b.Property("LlmSpendPeriod") + .HasColumnType("interval"); + + b.PrimitiveCollection("Operations") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.PrimitiveCollection>("Sites") + .IsRequired() + .HasColumnType("uuid[]"); + + b.PrimitiveCollection>("Types") + .HasColumnType("text[]"); + + b.HasKey("Id"); + + b.HasIndex("KeyHash") + .IsUnique(); + + b.ToTable("AgentCredentials", null, t => + { + t.HasCheckConstraint("CK_AgentCredentials_Caps", "\"CallsPerPeriod\" >= 1 AND \"ItemsPerChangeSet\" >= 1 AND \"ChangeSetsPerDay\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.ToolCallRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AgentCredentialId") + .HasColumnType("uuid"); + + b.Property("AgentName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Duration") + .HasColumnType("interval"); + + b.Property("Model") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Outcome") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("Tool") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("Id"); + + b.HasIndex("AgentCredentialId", "Timestamp"); + + b.ToTable("ToolCalls", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Audit.AuditEventRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AgentCredentialId") + .HasColumnType("uuid"); + + b.Property("ChangeSetId") + .HasColumnType("uuid"); + + b.Property("InitiatorId") + .HasColumnType("uuid"); + + b.Property("InitiatorKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("InitiatorName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Model") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Operation") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("PreviousVersionId") + .HasColumnType("uuid"); + + b.Property("PromptHash") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.PrimitiveCollection>("Sources") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("SubjectId") + .HasColumnType("uuid"); + + b.Property("SubjectType") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AgentCredentialId"); + + b.HasIndex("ChangeSetId"); + + b.HasIndex("Timestamp"); + + b.HasIndex("SubjectType", "SubjectId"); + + b.ToTable("AuditEvents", null, t => + { + t.HasCheckConstraint("CK_AuditEvents_AgentCredentialId", "(\"InitiatorKind\" = 'agent') = (\"AgentCredentialId\" IS NOT NULL)"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetOperationRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("BaseVersion") + .HasColumnType("integer"); + + b.Property("ChangeSetId") + .HasColumnType("uuid"); + + b.Property("ItemId") + .HasColumnType("uuid"); + + b.Property("ItemKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("Position") + .HasColumnType("integer"); + + b.Property("ProposedVersion") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ChangeSetId", "Position") + .IsUnique(); + + b.HasIndex("ItemKind", "ItemId"); + + b.HasIndex("ChangeSetId", "ItemKind", "ItemId") + .IsUnique(); + + b.ToTable("ChangeSetOperations", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("InitiatorId") + .HasColumnType("uuid"); + + b.Property("InitiatorKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("InitiatorName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Rationale") + .IsRequired() + .HasColumnType("text"); + + b.Property("RejectionReason") + .HasColumnType("text"); + + b.Property("RevertsChangeSetId") + .HasColumnType("uuid"); + + b.Property("Revision") + .HasColumnType("integer"); + + b.Property("SessionId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("SiteId") + .HasColumnType("uuid"); + + b.PrimitiveCollection>("Sources") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("State") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("RevertsChangeSetId"); + + b.HasIndex("SiteId"); + + b.HasIndex("State"); + + b.HasIndex("InitiatorKind", "InitiatorId", "SessionId") + .IsUnique() + .HasFilter("\"State\" = 'Open' AND \"SessionId\" IS NOT NULL"); + + b.ToTable("ChangeSets", null, t => + { + t.HasCheckConstraint("CK_ChangeSets_RejectionReason", "(\"State\" = 'Rejected') = (\"RejectionReason\" IS NOT NULL)"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("ActiveVersion") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SiteId") + .HasColumnType("uuid"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("SiteId", "Name") + .IsUnique(); + + b.ToTable("Checks", null, t => + { + t.HasCheckConstraint("CK_Checks_ActiveVersion", "\"ActiveVersion\" IS NULL OR (\"ActiveVersion\" >= 1 AND \"ActiveVersion\" <= \"Version\")"); + + t.HasCheckConstraint("CK_Checks_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckVersionRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CheckId") + .HasColumnType("uuid"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CheckId", "Version") + .IsUnique(); + + b.ToTable("CheckVersions", null, t => + { + t.HasCheckConstraint("CK_CheckVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryReferenceRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("EntryId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("TargetField") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("TargetId") + .HasColumnType("uuid"); + + b.Property("TargetKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("EntryId", "Version"); + + b.HasIndex("TargetKind", "TargetId", "TargetField"); + + b.ToTable("EntryReferences", null, t => + { + t.HasCheckConstraint("CK_EntryReferences_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("ContentTypeId") + .HasColumnType("uuid"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LatestVersion") + .HasColumnType("integer"); + + b.Property("PublishedVersion") + .HasColumnType("integer"); + + b.Property("SiteId") + .HasColumnType("uuid"); + + b.Property("State") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.HasKey("Id"); + + b.HasIndex("ContentTypeId"); + + b.HasIndex("SiteId", "ContentTypeId"); + + b.ToTable("Entries", null, t => + { + t.HasCheckConstraint("CK_Entries_LatestVersion", "\"LatestVersion\" >= 1"); + + t.HasCheckConstraint("CK_Entries_PublishedVersion", "\"PublishedVersion\" IS NULL OR (\"PublishedVersion\" >= 1 AND \"PublishedVersion\" <= \"LatestVersion\")"); + + t.HasCheckConstraint("CK_Entries_State", "\"State\" IN ('Draft', 'Published', 'Reverted', 'Archived')"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryVersionRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("ContentTypeVersion") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EntryId") + .HasColumnType("uuid"); + + b.Property("Fields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("EntryId", "Version") + .IsUnique(); + + b.ToTable("EntryVersions", null, t => + { + t.HasTrigger("TR_EntryVersions_NoUpdate"); + + t.HasCheckConstraint("CK_EntryVersions_ContentTypeVersion", "\"ContentTypeVersion\" >= 1"); + + t.HasCheckConstraint("CK_EntryVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Entities.EntityFactRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("EntryId") + .HasColumnType("uuid"); + + b.Property("Field") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("LastVerified") + .HasColumnType("timestamp with time zone"); + + b.Property("Owner") + .HasColumnType("uuid"); + + b.Property("Position") + .HasColumnType("integer"); + + b.Property("SourceKind") + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("SourceReference") + .HasColumnType("text"); + + b.Property("Unsourced") + .HasColumnType("boolean"); + + b.Property("ValidFrom") + .HasColumnType("timestamp with time zone"); + + b.Property("ValidUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("LastVerified"); + + b.HasIndex("Unsourced"); + + b.HasIndex("ValidUntil"); + + b.HasIndex("EntryId", "Version", "Field") + .IsUnique(); + + b.ToTable("EntityFacts", null, t => + { + t.HasCheckConstraint("CK_EntityFacts_Unsourced", "\"Unsourced\" = (\"SourceKind\" IS NULL)"); + + t.HasCheckConstraint("CK_EntityFacts_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Jobs.JobRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Attempts") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("InitiatorId") + .HasColumnType("uuid"); + + b.Property("InitiatorKind") + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("InitiatorName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastError") + .HasColumnType("text"); + + b.Property("LeaseOwner") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("MaxAttempts") + .HasColumnType("integer"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProgressDone") + .HasColumnType("integer"); + + b.Property("ProgressMessage") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("ProgressTotal") + .HasColumnType("integer"); + + b.Property("Revision") + .HasColumnType("integer"); + + b.Property("ScheduledAt") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("State") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("SubjectId") + .HasColumnType("uuid"); + + b.Property("SubjectType") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("Id"); + + b.HasIndex("State", "ScheduledAt"); + + b.HasIndex("SubjectType", "SubjectId"); + + b.ToTable("Jobs", null, t => + { + t.HasCheckConstraint("CK_Jobs_Attempts", "\"Attempts\" >= 0 AND \"MaxAttempts\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AgentCredentialId") + .HasColumnType("uuid"); + + b.Property("Cost") + .HasPrecision(18, 6) + .HasColumnType("numeric(18,6)"); + + b.Property("Duration") + .HasColumnType("interval"); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("InitiatorId") + .HasColumnType("uuid"); + + b.Property("InitiatorKind") + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("InputTokens") + .HasColumnType("bigint"); + + b.Property("Model") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("ModelVersion") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Outcome") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("OutputTokens") + .HasColumnType("bigint"); + + b.Property("Profile") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Prompt") + .HasColumnType("text"); + + b.Property("PromptHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("PurgedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Purpose") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Response") + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("PromptHash"); + + b.HasIndex("Timestamp"); + + b.HasIndex("AgentCredentialId", "Timestamp"); + + b.ToTable("ModelCalls", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Persistence.FluentCmsRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("Roles", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Persistence.FluentCmsUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AccessFailedCount") + .HasColumnType("integer"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("boolean"); + + b.Property("LockoutEnabled") + .HasColumnType("boolean"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PasswordHash") + .HasColumnType("text"); + + b.Property("PhoneNumber") + .HasColumnType("text"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("boolean"); + + b.Property("SecurityStamp") + .HasColumnType("text"); + + b.Property("TwoFactorEnabled") + .HasColumnType("boolean"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("Users", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("PublishedVersion") + .HasColumnType("integer"); + + b.Property("SiteId") + .HasColumnType("uuid"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("SiteId", "Slug") + .IsUnique(); + + b.ToTable("ContentTypes", null, t => + { + t.HasCheckConstraint("CK_ContentTypes_PublishedVersion", "\"PublishedVersion\" IS NULL OR \"PublishedVersion\" >= 1"); + + t.HasCheckConstraint("CK_ContentTypes_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeVersionRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("ContentTypeId") + .HasColumnType("uuid"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ContentTypeId", "Version") + .IsUnique(); + + b.ToTable("ContentTypeVersions", null, t => + { + t.HasCheckConstraint("CK_ContentTypeVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Sites.SiteRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("Id"); + + b.ToTable("Sites", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-0000-0000-000000000001"), + Name = "Default" + }); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("RoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("UserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("ProviderKey") + .HasColumnType("text"); + + b.Property("ProviderDisplayName") + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("UserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("UserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Value") + .HasColumnType("text"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("UserTokens", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.ToolCallRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) + .WithMany() + .HasForeignKey("AgentCredentialId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetOperationRow", b => + { + b.HasOne("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", null) + .WithMany() + .HasForeignKey("ChangeSetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", b => + { + b.HasOne("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", null) + .WithMany() + .HasForeignKey("RevertsChangeSetId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Checks.CheckRow", null) + .WithMany() + .HasForeignKey("CheckId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryReferenceRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Schema.ContentTypeRow", null) + .WithMany() + .HasForeignKey("ContentTypeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Entities.EntityFactRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) + .WithMany() + .HasForeignKey("AgentCredentialId") + .OnDelete(DeleteBehavior.Restrict); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Schema.ContentTypeRow", null) + .WithMany() + .HasForeignKey("ContentTypeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs new file mode 100644 index 000000000..843d165fc --- /dev/null +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs @@ -0,0 +1,44 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FluentCMS.Infrastructure.Persistence.Migrations.PostgreSql +{ + /// + public partial class SearchDocuments : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + // Outside the EF model: the tsvector is generated from the text, under the simple configuration so no language's + // stemming applies, and searched through a GIN index (ADR-0010). + migrationBuilder.Sql(""" + CREATE TABLE "SearchDocuments" ( + "EntryId" uuid NOT NULL PRIMARY KEY, + "Text" text NOT NULL, + "Document" tsvector GENERATED ALWAYS AS (to_tsvector('simple', "Text")) STORED + ); + CREATE INDEX "IX_SearchDocuments_Document" ON "SearchDocuments" USING GIN ("Document"); + """); + + // Every entry that already exists, flattened as SearchText does: each string and number of its latest version, as + // lowercase words. + migrationBuilder.Sql(""" + INSERT INTO "SearchDocuments" ("EntryId", "Text") + SELECT entry."Id", + trim(regexp_replace(lower(coalesce(( + SELECT string_agg(item #>> '{}', ' ') + FROM jsonb_path_query(version."Fields", 'strict $.**') AS item + WHERE jsonb_typeof(item) IN ('string', 'number')), '')), '[^[:alnum:]]+', ' ', 'g')) + FROM "Entries" AS entry + JOIN "EntryVersions" AS version ON version."EntryId" = entry."Id" AND version."Version" = entry."LatestVersion"; + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("""DROP TABLE "SearchDocuments";"""); + } + } +} diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs new file mode 100644 index 000000000..2e13c5498 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs @@ -0,0 +1,1179 @@ +// +using System; +using FluentCMS.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace FluentCMS.Infrastructure.Persistence.Migrations.Sqlite +{ + [DbContext(typeof(SqliteFluentCmsDbContext))] + [Migration("20261019033130_SearchDocuments")] + partial class SearchDocuments + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.13"); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.AgentCredentialRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("CallPeriod") + .HasColumnType("TEXT"); + + b.Property("CallsPerPeriod") + .HasColumnType("INTEGER"); + + b.Property("ChangeSetsPerDay") + .HasColumnType("INTEGER"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("ExpiresAt") + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Fields") + .HasColumnType("TEXT"); + + b.Property("IssuedAt") + .HasColumnType("TEXT"); + + b.Property("ItemsPerChangeSet") + .HasColumnType("INTEGER"); + + b.Property("KeyHash") + .IsRequired() + .HasColumnType("BLOB"); + + b.Property("LlmSpendPerPeriod") + .HasColumnType("TEXT"); + + b.Property("LlmSpendPeriod") + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Operations") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RevokedAt") + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Sites") + .IsRequired() + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Types") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("KeyHash") + .IsUnique(); + + b.ToTable("AgentCredentials", null, t => + { + t.HasCheckConstraint("CK_AgentCredentials_Caps", "\"CallsPerPeriod\" >= 1 AND \"ItemsPerChangeSet\" >= 1 AND \"ChangeSetsPerDay\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.ToolCallRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AgentCredentialId") + .HasColumnType("TEXT"); + + b.Property("AgentName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("Duration") + .HasColumnType("TEXT"); + + b.Property("Model") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Outcome") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.Property("Tool") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AgentCredentialId", "Timestamp"); + + b.ToTable("ToolCalls", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Audit.AuditEventRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AgentCredentialId") + .HasColumnType("TEXT"); + + b.Property("ChangeSetId") + .HasColumnType("TEXT"); + + b.Property("InitiatorId") + .HasColumnType("TEXT"); + + b.Property("InitiatorKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("InitiatorName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("Model") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Operation") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("PreviousVersionId") + .HasColumnType("TEXT"); + + b.Property("PromptHash") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Sources") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SubjectId") + .HasColumnType("TEXT"); + + b.Property("SubjectType") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AgentCredentialId"); + + b.HasIndex("ChangeSetId"); + + b.HasIndex("Timestamp"); + + b.HasIndex("SubjectType", "SubjectId"); + + b.ToTable("AuditEvents", null, t => + { + t.HasCheckConstraint("CK_AuditEvents_AgentCredentialId", "(\"InitiatorKind\" = 'agent') = (\"AgentCredentialId\" IS NOT NULL)"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetOperationRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("BaseVersion") + .HasColumnType("INTEGER"); + + b.Property("ChangeSetId") + .HasColumnType("TEXT"); + + b.Property("ItemId") + .HasColumnType("TEXT"); + + b.Property("ItemKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("Position") + .HasColumnType("INTEGER"); + + b.Property("ProposedVersion") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("ChangeSetId", "Position") + .IsUnique(); + + b.HasIndex("ItemKind", "ItemId"); + + b.HasIndex("ChangeSetId", "ItemKind", "ItemId") + .IsUnique(); + + b.ToTable("ChangeSetOperations", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("InitiatorId") + .HasColumnType("TEXT"); + + b.Property("InitiatorKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("InitiatorName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("Rationale") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RejectionReason") + .HasColumnType("TEXT"); + + b.Property("RevertsChangeSetId") + .HasColumnType("TEXT"); + + b.Property("Revision") + .HasColumnType("INTEGER"); + + b.Property("SessionId") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("SiteId") + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Sources") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("State") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("RevertsChangeSetId"); + + b.HasIndex("SiteId"); + + b.HasIndex("State"); + + b.HasIndex("InitiatorKind", "InitiatorId", "SessionId") + .IsUnique() + .HasFilter("\"State\" = 'Open' AND \"SessionId\" IS NOT NULL"); + + b.ToTable("ChangeSets", null, t => + { + t.HasCheckConstraint("CK_ChangeSets_RejectionReason", "(\"State\" = 'Rejected') = (\"RejectionReason\" IS NOT NULL)"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("ActiveVersion") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SiteId") + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("SiteId", "Name") + .IsUnique(); + + b.ToTable("Checks", null, t => + { + t.HasCheckConstraint("CK_Checks_ActiveVersion", "\"ActiveVersion\" IS NULL OR (\"ActiveVersion\" >= 1 AND \"ActiveVersion\" <= \"Version\")"); + + t.HasCheckConstraint("CK_Checks_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckVersionRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("CheckId") + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("CheckId", "Version") + .IsUnique(); + + b.ToTable("CheckVersions", null, t => + { + t.HasCheckConstraint("CK_CheckVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryReferenceRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("EntryId") + .HasColumnType("TEXT"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("TargetField") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TargetId") + .HasColumnType("TEXT"); + + b.Property("TargetKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("EntryId", "Version"); + + b.HasIndex("TargetKind", "TargetId", "TargetField"); + + b.ToTable("EntryReferences", null, t => + { + t.HasCheckConstraint("CK_EntryReferences_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("ContentTypeId") + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("LatestVersion") + .HasColumnType("INTEGER"); + + b.Property("PublishedVersion") + .HasColumnType("INTEGER"); + + b.Property("SiteId") + .HasColumnType("TEXT"); + + b.Property("State") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ContentTypeId"); + + b.HasIndex("SiteId", "ContentTypeId"); + + b.ToTable("Entries", null, t => + { + t.HasCheckConstraint("CK_Entries_LatestVersion", "\"LatestVersion\" >= 1"); + + t.HasCheckConstraint("CK_Entries_PublishedVersion", "\"PublishedVersion\" IS NULL OR (\"PublishedVersion\" >= 1 AND \"PublishedVersion\" <= \"LatestVersion\")"); + + t.HasCheckConstraint("CK_Entries_State", "\"State\" IN ('Draft', 'Published', 'Reverted', 'Archived')"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryVersionRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("ContentTypeVersion") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("EntryId") + .HasColumnType("TEXT"); + + b.Property("Fields") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("EntryId", "Version") + .IsUnique(); + + b.ToTable("EntryVersions", null, t => + { + t.HasTrigger("TR_EntryVersions_NoUpdate"); + + t.HasCheckConstraint("CK_EntryVersions_ContentTypeVersion", "\"ContentTypeVersion\" >= 1"); + + t.HasCheckConstraint("CK_EntryVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Entities.EntityFactRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("EntryId") + .HasColumnType("TEXT"); + + b.Property("Field") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("LastVerified") + .HasColumnType("TEXT"); + + b.Property("Owner") + .HasColumnType("TEXT"); + + b.Property("Position") + .HasColumnType("INTEGER"); + + b.Property("SourceKind") + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("SourceReference") + .HasColumnType("TEXT"); + + b.Property("Unsourced") + .HasColumnType("INTEGER"); + + b.Property("ValidFrom") + .HasColumnType("TEXT"); + + b.Property("ValidUntil") + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("LastVerified"); + + b.HasIndex("Unsourced"); + + b.HasIndex("ValidUntil"); + + b.HasIndex("EntryId", "Version", "Field") + .IsUnique(); + + b.ToTable("EntityFacts", null, t => + { + t.HasCheckConstraint("CK_EntityFacts_Unsourced", "\"Unsourced\" = (\"SourceKind\" IS NULL)"); + + t.HasCheckConstraint("CK_EntityFacts_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Jobs.JobRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Attempts") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("FinishedAt") + .HasColumnType("TEXT"); + + b.Property("InitiatorId") + .HasColumnType("TEXT"); + + b.Property("InitiatorKind") + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("InitiatorName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasColumnType("TEXT"); + + b.Property("LeaseOwner") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("LeaseUntil") + .HasColumnType("TEXT"); + + b.Property("MaxAttempts") + .HasColumnType("INTEGER"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ProgressDone") + .HasColumnType("INTEGER"); + + b.Property("ProgressMessage") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ProgressTotal") + .HasColumnType("INTEGER"); + + b.Property("Revision") + .HasColumnType("INTEGER"); + + b.Property("ScheduledAt") + .HasColumnType("TEXT"); + + b.Property("StartedAt") + .HasColumnType("TEXT"); + + b.Property("State") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("SubjectId") + .HasColumnType("TEXT"); + + b.Property("SubjectType") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("State", "ScheduledAt"); + + b.HasIndex("SubjectType", "SubjectId"); + + b.ToTable("Jobs", null, t => + { + t.HasCheckConstraint("CK_Jobs_Attempts", "\"Attempts\" >= 0 AND \"MaxAttempts\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AgentCredentialId") + .HasColumnType("TEXT"); + + b.Property("Cost") + .HasPrecision(18, 6) + .HasColumnType("TEXT"); + + b.Property("Duration") + .HasColumnType("TEXT"); + + b.Property("Error") + .HasColumnType("TEXT"); + + b.Property("InitiatorId") + .HasColumnType("TEXT"); + + b.Property("InitiatorKind") + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("InputTokens") + .HasColumnType("INTEGER"); + + b.Property("Model") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ModelVersion") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Outcome") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT"); + + b.Property("OutputTokens") + .HasColumnType("INTEGER"); + + b.Property("Profile") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Prompt") + .HasColumnType("TEXT"); + + b.Property("PromptHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("PurgedAt") + .HasColumnType("TEXT"); + + b.Property("Purpose") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("Response") + .HasColumnType("TEXT"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("PromptHash"); + + b.HasIndex("Timestamp"); + + b.HasIndex("AgentCredentialId", "Timestamp"); + + b.ToTable("ModelCalls", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Persistence.FluentCmsRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("Roles", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Persistence.FluentCmsUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("AccessFailedCount") + .HasColumnType("INTEGER"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("EmailConfirmed") + .HasColumnType("INTEGER"); + + b.Property("LockoutEnabled") + .HasColumnType("INTEGER"); + + b.Property("LockoutEnd") + .HasColumnType("TEXT"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("PasswordHash") + .HasColumnType("TEXT"); + + b.Property("PhoneNumber") + .HasColumnType("TEXT"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("INTEGER"); + + b.Property("SecurityStamp") + .HasColumnType("TEXT"); + + b.Property("TwoFactorEnabled") + .HasColumnType("INTEGER"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("Users", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("PublishedVersion") + .HasColumnType("INTEGER"); + + b.Property("SiteId") + .HasColumnType("TEXT"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("SiteId", "Slug") + .IsUnique(); + + b.ToTable("ContentTypes", null, t => + { + t.HasCheckConstraint("CK_ContentTypes_PublishedVersion", "\"PublishedVersion\" IS NULL OR \"PublishedVersion\" >= 1"); + + t.HasCheckConstraint("CK_ContentTypes_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeVersionRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("ContentTypeId") + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("ContentTypeId", "Version") + .IsUnique(); + + b.ToTable("ContentTypeVersions", null, t => + { + t.HasCheckConstraint("CK_ContentTypeVersions_Version", "\"Version\" >= 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Sites.SiteRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Sites", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-0000-0000-000000000001"), + Name = "Default" + }); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("RoleId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("RoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("UserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("ProviderKey") + .HasColumnType("TEXT"); + + b.Property("ProviderDisplayName") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("TEXT"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("UserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("TEXT"); + + b.Property("RoleId") + .HasColumnType("TEXT"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("UserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("TEXT"); + + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("UserTokens", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Agents.ToolCallRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) + .WithMany() + .HasForeignKey("AgentCredentialId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetOperationRow", b => + { + b.HasOne("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", null) + .WithMany() + .HasForeignKey("ChangeSetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", b => + { + b.HasOne("FluentCMS.Infrastructure.ChangeSets.ChangeSetRow", null) + .WithMany() + .HasForeignKey("RevertsChangeSetId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Checks.CheckVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Checks.CheckRow", null) + .WithMany() + .HasForeignKey("CheckId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryReferenceRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Schema.ContentTypeRow", null) + .WithMany() + .HasForeignKey("ContentTypeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Content.EntryVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Entities.EntityFactRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Content.EntryRow", null) + .WithMany() + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) + .WithMany() + .HasForeignKey("AgentCredentialId") + .OnDelete(DeleteBehavior.Restrict); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Schema.ContentTypeVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Schema.ContentTypeRow", null) + .WithMany() + .HasForeignKey("ContentTypeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("FluentCMS.Infrastructure.Persistence.FluentCmsUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs new file mode 100644 index 000000000..485de72e4 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs @@ -0,0 +1,39 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FluentCMS.Infrastructure.Persistence.Migrations.Sqlite +{ + /// + public partial class SearchDocuments : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + // Outside the EF model: an FTS5 table (ADR-0010). Diacritics are kept, as PostgreSQL's simple configuration keeps them, + // so both providers match the same words. + migrationBuilder.Sql(""" + CREATE VIRTUAL TABLE "SearchDocuments" USING fts5("EntryId" UNINDEXED, "Text", tokenize = 'unicode61 remove_diacritics 0'); + """); + + // Every entry that already exists, flattened as SearchText does: each string and number of its latest version. The + // tokenizer splits on everything but letters and digits and folds case, as SearchText does before indexing. + migrationBuilder.Sql(""" + INSERT INTO "SearchDocuments" ("EntryId", "Text") + SELECT entry."Id", + coalesce(( + SELECT group_concat(item.value, ' ') + FROM json_tree(version."Fields") AS item + WHERE item.type IN ('text', 'integer', 'real')), '') + FROM "Entries" AS entry + JOIN "EntryVersions" AS version ON version."EntryId" = entry."Id" AND version."Version" = entry."LatestVersion"; + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("""DROP TABLE "SearchDocuments";"""); + } + } +} diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs new file mode 100644 index 000000000..bf1ec39e8 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -0,0 +1,135 @@ +using FluentCMS.Core.Agents; +using FluentCMS.Core.Content; +using FluentCMS.Core.Results; +using FluentCMS.Core.Search; +using FluentCMS.Core.Sites; +using FluentCMS.Infrastructure.Content; +using FluentCMS.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; + +namespace FluentCMS.Infrastructure.Search; + +// The index finds the text; the entry rows decide the rest, so deleting, restoring and publishing change results without touching +// the index. A match is followed to the entries whose latest version references it, such as the page that shows a matching entity. +internal sealed class EntrySearch(FluentCmsDbContext context, ISearchIndex index, IAgentPermissions permissions) : ISearch +{ + // Matches read before filtering, so filters that drop many still leave enough to fill the limit. + private const int CandidatesPerHit = 10; + private const int MaxCandidates = 1000; + + public async Task> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + + var terms = SearchTerms.From(query.Text); + var invalid = Validate(query, terms); + if (invalid.Count > 0) + { + return Result.Failure(invalid); + } + + var readable = await permissions.ReadableTypesAsync(cancellationToken); + var matches = await index.MatchAsync(terms, Math.Min(query.Limit * CandidatesPerHit, MaxCandidates), cancellationToken); + var ranks = matches.ToDictionary(match => match.EntryId, match => match.Rank); + var matchedIds = ranks.Keys.ToList(); + + var matched = (await EntriesAsync(context.Entries.Where(row => matchedIds.Contains(row.Id)), cancellationToken)) + .Where(entry => readable(entry.ContentType)) + .ToList(); + var direct = matched + .Where(entry => Passes(entry, query)) + .Select(entry => entry.Hit(ranks[entry.Row.Id], via: null)) + .OrderByDescending(hit => hit.Rank) + .ThenBy(hit => hit.EntryId) + .Take(query.Limit) + .ToList(); + + var room = query.Limit - direct.Count; + if (room == 0) + { + return Result.Success(new SearchResults(direct)); + } + + // Only matches the caller may read lead anywhere, so a reference never reveals an entry of a type outside the scope. + var targets = matched.ToDictionary(entry => entry.Row.Id, entry => ranks[entry.Row.Id]); + var targetIds = targets.Keys.ToList(); + var references = await context.EntryReferences + .AsNoTracking() + .Where(reference => targetIds.Contains(reference.TargetId)) + .Join(context.Entries, reference => reference.EntryId, row => row.Id, (reference, row) => new { reference, row }) + .Where(pair => pair.reference.Version == pair.row.LatestVersion) + .Select(pair => new { pair.reference.EntryId, pair.reference.TargetId }) + .ToListAsync(cancellationToken); + + var referencingIds = references.Select(reference => reference.EntryId).Distinct().ToList(); + var shown = direct.Select(hit => hit.EntryId).ToHashSet(); + var via = (await EntriesAsync(context.Entries.Where(row => referencingIds.Contains(row.Id)), cancellationToken)) + .Where(entry => !shown.Contains(entry.Row.Id) && readable(entry.ContentType) && Passes(entry, query)) + .Select(entry => + { + // The best match the entry references explains why it is shown. + var target = references + .Where(reference => reference.EntryId == entry.Row.Id) + .Select(reference => reference.TargetId) + .OrderByDescending(id => targets[id]) + .ThenBy(id => id) + .First(); + return entry.Hit(targets[target], target); + }) + .OrderByDescending(hit => hit.Rank) + .ThenBy(hit => hit.EntryId) + .Take(room); + + return Result.Success(new SearchResults([.. direct, .. via])); + } + + private async Task> EntriesAsync(IQueryable rows, CancellationToken cancellationToken) + { + var siteId = Site.Default.Id; + var found = await rows + .AsNoTracking() + .Where(row => row.SiteId == siteId && row.DeletedAt == null) + .Join(context.ContentTypes, row => row.ContentTypeId, type => type.Id, (row, type) => new { row, type.Slug }) + .ToListAsync(cancellationToken); + + return [.. found.Select(pair => new Candidate(pair.row, pair.Slug))]; + } + + private static bool Passes(Candidate entry, SearchQuery query) + { + return (query.ContentTypes is null || query.ContentTypes.Contains(entry.ContentType, StringComparer.Ordinal)) + && (query.States is null || query.States.Contains(entry.Row.State)) + && (!query.PublishedOnly || entry.Row.PublishedVersion is not null); + } + + private static List Validate(SearchQuery query, IReadOnlyList terms) + { + var errors = new List(); + + if (terms.Count == 0) + { + errors.Add(new Error(SearchErrorCodes.QueryEmpty, "The query has no words to search for; give at least one word or number, such as '4411'.")); + } + else if (terms.Count > SearchTerms.MaxTerms || terms.Any(term => term.Length > SearchTerms.MaxTermLength)) + { + errors.Add(new Error( + SearchErrorCodes.QueryTooLong, + $"A query has at most {SearchTerms.MaxTerms} words of at most {SearchTerms.MaxTermLength} characters each; search for fewer, shorter words.")); + } + + if (query.Limit is < 1 or > SearchQuery.MaxLimit) + { + errors.Add(new Error(SearchErrorCodes.LimitInvalid, $"The limit is from 1 to {SearchQuery.MaxLimit}, and {query.Limit} was given.")); + } + + return errors; + } + + private sealed record Candidate(EntryRow Row, string ContentType) + { + public SearchHit Hit(double rank, Guid? via) + { + return new SearchHit(Row.Id, ContentType, Row.State, Row.LatestVersion, Row.PublishedVersion, rank, via); + } + } +} diff --git a/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs b/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs new file mode 100644 index 000000000..3d03ce420 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs @@ -0,0 +1,79 @@ +using FluentCMS.Core.Search; +using FluentCMS.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; + +namespace FluentCMS.Infrastructure.Search; + +// The SearchDocuments table is outside the EF model, because each provider stores it its own way (ADR-0010); both are written with +// SQL inside the current transaction, so an index change commits or rolls back with the write it reflects. The text is already +// lowercase words (SearchText), so both tokenizers only split it on spaces. +internal sealed class PostgreSqlSearchIndex(FluentCmsDbContext context) : ISearchIndex +{ + public async Task IndexAsync(SearchDocument document, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + + await context.Database.ExecuteSqlAsync( + $""" + INSERT INTO "SearchDocuments" ("EntryId", "Text") VALUES ({document.EntryId}, {document.Text}) + ON CONFLICT ("EntryId") DO UPDATE SET "Text" = EXCLUDED."Text" + """, + cancellationToken); + } + + public async Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(terms); + + // Every term as a prefix; terms are letters and digits only, so nothing in them is tsquery syntax. + var query = string.Join(" & ", terms.Select(term => term + ":*")); + var rows = await context.Database + .SqlQuery( + $""" + SELECT "EntryId", ts_rank("Document", matched)::float8 AS "Rank" + FROM "SearchDocuments", to_tsquery('simple', {query}) AS matched + WHERE "Document" @@ matched + ORDER BY "Rank" DESC, "EntryId" + LIMIT {limit} + """) + .ToListAsync(cancellationToken); + + return [.. rows.Select(row => new SearchMatch(row.EntryId, row.Rank))]; + } +} + +internal sealed class SqliteSearchIndex(FluentCmsDbContext context) : ISearchIndex +{ + public async Task IndexAsync(SearchDocument document, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + + // An FTS5 table has no unique key besides its rowid, so the entry's row is replaced. + await context.Database.ExecuteSqlAsync($"""DELETE FROM "SearchDocuments" WHERE "EntryId" = {document.EntryId}""", cancellationToken); + await context.Database.ExecuteSqlAsync( + $"""INSERT INTO "SearchDocuments" ("EntryId", "Text") VALUES ({document.EntryId}, {document.Text})""", + cancellationToken); + } + + public async Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(terms); + + // Every term as a quoted prefix; terms are letters and digits only, so nothing in them is FTS5 syntax. + var query = string.Join(" AND ", terms.Select(term => $"\"{term}\"*")); + var rows = await context.Database + .SqlQuery( + $""" + SELECT "EntryId", -bm25("SearchDocuments") AS "Rank" + FROM "SearchDocuments" + WHERE "SearchDocuments" MATCH {query} + ORDER BY bm25("SearchDocuments"), "EntryId" + LIMIT {limit} + """) + .ToListAsync(cancellationToken); + + return [.. rows.Select(row => new SearchMatch(row.EntryId, row.Rank))]; + } +} + +internal sealed record MatchRow(Guid EntryId, double Rank); diff --git a/tests/FluentCMS.IntegrationTests/Mcp/BuiltInToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/BuiltInToolsTests.cs index 9728b8f42..7e7b23f4e 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/BuiltInToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/BuiltInToolsTests.cs @@ -37,7 +37,7 @@ public async Task An_agent_reads_content_types_and_the_schema_document_with_the_ { Assert.False(string.IsNullOrWhiteSpace(tool.Description)); Assert.Equal("object", tool.ProtocolTool.InputSchema.GetProperty("type").GetString()); - Assert.Equal(tool.Name.StartsWith("get_", StringComparison.Ordinal) || tool.Name.StartsWith("list_", StringComparison.Ordinal), tool.ProtocolTool.Annotations?.ReadOnlyHint); + Assert.Equal(new[] { "get_", "list_", "search_" }.Any(verb => tool.Name.StartsWith(verb, StringComparison.Ordinal)), tool.ProtocolTool.Annotations?.ReadOnlyHint); }); var list = await CallAsync(client, "list_content_types"); diff --git a/tests/FluentCMS.IntegrationTests/Mcp/McpAgents.cs b/tests/FluentCMS.IntegrationTests/Mcp/McpAgents.cs index 2866d77e3..4fd47a221 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/McpAgents.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/McpAgents.cs @@ -24,7 +24,8 @@ internal static class McpAgents "check_draft", "create_content_type", "create_entry", "delete_entry", "get_blast_radius", "get_change_set", "get_change_set_diff", "get_content_type", "get_entity_facts", "get_entity_relations", "get_entry", "get_entry_history", "get_resolved_entry", "get_schema_document", "get_where_used", "list_content_types", "list_entries", "list_stale_facts", "list_unsourced_facts", - "open_change_set", "relate_entities", "reopen_change_set", "revise_content_type", "set_entity_fact", "submit_change_set", + "open_change_set", "relate_entities", "reopen_change_set", "revise_content_type", "search_entries", "set_entity_fact", + "submit_change_set", "update_entry", ]; diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs new file mode 100644 index 000000000..77a8288fd --- /dev/null +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -0,0 +1,48 @@ +using System.Text.Json; +using FluentCMS.Core.Content; +using FluentCMS.Core.Search; +using FluentCMS.Core.Tools; +using FluentCMS.Infrastructure.Persistence; + +namespace FluentCMS.IntegrationTests.Mcp; + +public sealed class SearchToolsTests +{ + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task An_agent_finds_entries_of_the_types_it_may_read_and_no_others(string provider) + { + await using var agent = await ConnectedAgent.StartAsync(provider, types: ["page"]); + await agent.CreateAsPersonAsync("page"); + await agent.CreateAsPersonAsync("product"); + var page = await CreateAsync(agent, "page", "Spring sale on part 4411"); + await CreateAsync(agent, "product", "Part 4411"); + + var found = await agent.CallAsync("search_entries", new() { ["query"] = "4411" }); + var drafts = await agent.CallAsync("search_entries", new() { ["query"] = "4411", ["states"] = new[] { "draft" }, ["limit"] = 5 }); + + var hit = Assert.Single(found["hits"]!.AsArray())!; + Assert.Equal((page.ToString(), "page", "draft", (string?)null), ((string?)hit["id"], (string?)hit["contentType"], (string?)hit["state"], (string?)hit["via"])); + Assert.Single(drafts["hits"]!.AsArray()); + } + + [Fact] + public async Task A_query_without_words_or_an_unknown_state_is_a_tool_error() + { + await using var agent = await ConnectedAgent.StartAsync(DatabaseProviders.Sqlite); + + var empty = await agent.FailAsync("search_entries", new() { ["query"] = "?!" }); + var state = await agent.FailAsync("search_entries", new() { ["query"] = "4411", ["states"] = new[] { "live" } }); + + Assert.Equal(SearchErrorCodes.QueryEmpty, (string?)empty["errors"]![0]!["code"]); + Assert.Equal(ToolErrorCodes.ArgumentInvalid, (string?)state["errors"]![0]!["code"]); + Assert.Contains("'live' is not an entry state", (string?)state["errors"]![0]!["message"]); + } + + private static async Task CreateAsync(ConnectedAgent agent, string type, string title) + { + var fields = JsonSerializer.SerializeToElement(new { title }); + return (await agent.AsPersonAsync(entries => entries.CreateAsync(new NewEntry(type, fields)))).Id; + } +} diff --git a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs new file mode 100644 index 000000000..c6ef33414 --- /dev/null +++ b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs @@ -0,0 +1,149 @@ +using FluentCMS.Core.Content; +using FluentCMS.Core.Persistence; +using FluentCMS.Core.Results; +using FluentCMS.Core.Search; +using FluentCMS.Infrastructure.Persistence; +using FluentCMS.IntegrationTests.Content; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using static FluentCMS.IntegrationTests.Content.EntryScenario; +using static FluentCMS.IntegrationTests.Schema.SchemaScenario; + +namespace FluentCMS.IntegrationTests.Search; + +public sealed class SearchTests +{ + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_sku_finds_the_entity_and_the_page_that_references_it_and_filters_narrow_the_result(string provider) + { + await using var scenario = await CatalogAsync(provider); + var (product, page, other) = (scenario.Product, scenario.Page, scenario.Other); + + var all = await SearchAsync(scenario.Entries, new SearchQuery("4411")); + var prefix = await SearchAsync(scenario.Entries, new SearchQuery("441")); + var pagesOnly = await SearchAsync(scenario.Entries, new SearchQuery("4411") { ContentTypes = ["page"] }); + var twoWords = await SearchAsync(scenario.Entries, new SearchQuery("widget 4411")); + var publishedBefore = await SearchAsync(scenario.Entries, new SearchQuery("4411") { PublishedOnly = true }); + await scenario.Entries.Schema.QueryAsync(context => context.Entries.Where(row => row.Id == product).ExecuteUpdateAsync(setters => setters.SetProperty(row => row.PublishedVersion, 1))); + var publishedAfter = await SearchAsync(scenario.Entries, new SearchQuery("4411") { PublishedOnly = true }); + + Assert.Equal([(product, (Guid?)null), (page, product)], all.Select(hit => (hit.EntryId, hit.Via))); + Assert.Equal(("product", EntryState.Draft, 1), (all[0].ContentType, all[0].State, all[0].LatestVersion)); + Assert.Equal(all.Select(hit => hit.EntryId), prefix.Select(hit => hit.EntryId)); + Assert.Equal([page], pagesOnly.Select(hit => hit.EntryId)); + Assert.Equal([product, page], twoWords.Select(hit => hit.EntryId)); + Assert.DoesNotContain(all, hit => hit.EntryId == other); + Assert.Empty(publishedBefore); + Assert.Equal([product], publishedAfter.Select(hit => hit.EntryId)); + } + + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task An_entry_is_found_once_its_write_commits_and_not_after_a_rollback_or_while_deleted(string provider) + { + await using var scenario = await EntryScenario.CreateAsync(provider); + await scenario.Schema.CreateAsync("note", Field("text", "text")); + + await using (var scope = scenario.Services.CreateAsyncScope()) + { + await using var work = await scope.ServiceProvider.GetRequiredService().BeginAsync(); + Assert.True((await scope.ServiceProvider.GetRequiredService().CreateAsync(new NewEntry("note", Json("""{"text": "rolled back aurora"}""")))).IsSuccess); + } + + var note = (await scenario.CreateAsync("note", """{"text": "first draft borealis"}""")).Value; + var found = await SearchAsync(scenario, new SearchQuery("borealis")); + var rolledBack = await SearchAsync(scenario, new SearchQuery("aurora")); + + await scenario.UpdateAsync(note.Id, 1, """{"text": "second draft australis"}"""); + var oldWord = await SearchAsync(scenario, new SearchQuery("borealis")); + var newWord = await SearchAsync(scenario, new SearchQuery("australis")); + + Assert.True((await scenario.RunAsync(entries => entries.DeleteAsync(note.Id, 2))).IsSuccess); + var deleted = await SearchAsync(scenario, new SearchQuery("australis")); + await scenario.RunAsync(entries => entries.RestoreAsync(note.Id)); + var restored = await SearchAsync(scenario, new SearchQuery("australis")); + + Assert.Equal([note.Id], found.Select(hit => hit.EntryId)); + Assert.Empty(rolledBack); + Assert.Empty(oldWord); + Assert.Equal([note.Id], newWord.Select(hit => hit.EntryId)); + Assert.Empty(deleted); + Assert.Equal([note.Id], restored.Select(hit => hit.EntryId)); + } + + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task Entries_that_existed_before_the_index_are_found_after_the_migration(string provider) + { + await using var scenario = await CatalogAsync(provider); + + await using (var scope = scenario.Entries.Services.CreateAsyncScope()) + { + var database = scope.ServiceProvider.GetRequiredService().Database; + var migrations = database.GetMigrations().ToList(); + var index = migrations.FindIndex(name => name.EndsWith("_SearchDocuments", StringComparison.Ordinal)); + await database.GetService().MigrateAsync(migrations[index - 1]); + await database.MigrateAsync(); + } + + var hits = await SearchAsync(scenario.Entries, new SearchQuery("widget 4411")); + + Assert.Equal([scenario.Product, scenario.Page], hits.Select(hit => hit.EntryId)); + } + + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_query_without_words_or_with_too_many_or_a_limit_out_of_range_is_refused(string provider) + { + await using var scenario = await EntryScenario.CreateAsync(provider); + + var empty = await RunAsync(scenario, new SearchQuery(" -- ")); + var tooMany = await RunAsync(scenario, new SearchQuery(string.Join(' ', Enumerable.Range(1, SearchTerms.MaxTerms + 1).Select(word => $"w{word}")))); + var limit = await RunAsync(scenario, new SearchQuery("widget") { Limit = 0 }); + + Assert.Equal(SearchErrorCodes.QueryEmpty, Assert.Single(empty.Errors).Code); + Assert.Equal(SearchErrorCodes.QueryTooLong, Assert.Single(tooMany.Errors).Code); + Assert.Equal(SearchErrorCodes.LimitInvalid, Assert.Single(limit.Errors).Code); + } + + // A product entity with a SKU, a page that references it, and an unrelated page. + private static async Task CatalogAsync(string provider) + { + var entries = await EntryScenario.CreateAsync(provider); + await entries.Schema.CreateEntityAsync("product", Field("name", "text"), Field("sku", "text")); + await entries.Schema.CreateAsync("page", Field("title", "text"), Reference("product", "product")); + + var product = (await entries.CreateAsync("product", $$"""{"name": {{Fact("\"Widget\"")}}, "sku": {{Fact("\"A-4411\"")}}}""")).Value.Id; + var page = (await entries.CreateAsync("page", $$"""{"title": "Our best seller", "product": "{{product}}"}""")).Value.Id; + var other = (await entries.CreateAsync("page", """{"title": "Opening hours", "product": null}""")).Value.Id; + return new Catalog(entries, product, page, other); + } + + private static async Task> SearchAsync(EntryScenario scenario, SearchQuery query) + { + var results = await RunAsync(scenario, query); + Assert.True(results.IsSuccess, string.Join("; ", results.Errors.Select(error => error.Message))); + return results.Value.Hits; + } + + private static async Task> RunAsync(EntryScenario scenario, SearchQuery query) + { + await using var scope = scenario.Services.CreateAsyncScope(); + return await scope.ServiceProvider.GetRequiredService().SearchAsync(query); + } + + private sealed record Catalog(EntryScenario Entries, Guid Product, Guid Page, Guid Other) : IAsyncDisposable + { + public ValueTask DisposeAsync() + { + return Entries.DisposeAsync(); + } + } +} diff --git a/tests/FluentCMS.Tests/Search/SearchTextTests.cs b/tests/FluentCMS.Tests/Search/SearchTextTests.cs new file mode 100644 index 000000000..1e890dfb3 --- /dev/null +++ b/tests/FluentCMS.Tests/Search/SearchTextTests.cs @@ -0,0 +1,34 @@ +using System.Text.Json; +using FluentCMS.Core.Search; + +namespace FluentCMS.Tests.Search; + +public sealed class SearchTextTests +{ + [Fact] + public void An_entrys_text_is_every_string_and_number_in_its_fields_as_lowercase_words() + { + var fields = JsonDocument.Parse(""" + { + "title": "Widget Pro", + "sku": {"value": "A-4411", "source": {"kind": "url", "reference": "https://example.com/spec"}, "owner": null}, + "price": 19.99, + "tags": ["Garden", "Café"], + "active": true + } + """).RootElement; + + Assert.Equal("widget pro a 4411 url https example com spec 19 99 garden café", SearchText.From(fields)); + } + + [Theory] + [InlineData("4411", new[] { "4411" })] + [InlineData(" Widget A-4411 widget ", new[] { "widget", "a", "4411" })] + [InlineData("19.99 €", new[] { "19", "99" })] + [InlineData("-- ?! ", new string[0])] + [InlineData(null, new string[0])] + public void A_query_is_split_into_the_same_words_as_the_text(string? query, string[] terms) + { + Assert.Equal(terms, SearchTerms.From(query)); + } +} diff --git a/tests/FluentCMS.Tests/Tools/McpCoverageTests.cs b/tests/FluentCMS.Tests/Tools/McpCoverageTests.cs index f191a0da9..b0f771c61 100644 --- a/tests/FluentCMS.Tests/Tools/McpCoverageTests.cs +++ b/tests/FluentCMS.Tests/Tools/McpCoverageTests.cs @@ -36,7 +36,7 @@ public sealed class McpCoverageTests Missing("CHK", "Read the check results of a change set", "#2420"), Missing("DSK", "Read review decisions, read-only", "#2422"), Missing("MED", "Upload and describe media", "#2427"), - Missing("Search", "Search entries and entities", "#2430"), + new("Search", "Search entries and entities by their words", ["search_entries"]), Missing("MCP-08", "Bulk operations that return a change set id and report progress", "#2423"), ]; diff --git a/tests/FluentCMS.Tests/Tools/ToolProviderConformanceTests.cs b/tests/FluentCMS.Tests/Tools/ToolProviderConformanceTests.cs index df9c5b2a0..9a1c938f1 100644 --- a/tests/FluentCMS.Tests/Tools/ToolProviderConformanceTests.cs +++ b/tests/FluentCMS.Tests/Tools/ToolProviderConformanceTests.cs @@ -124,3 +124,8 @@ public sealed class EntityToolsConformanceTests : ToolProviderConformanceTests { protected override object Provider { get; } = new EntityTools(); } + +public sealed class SearchToolsConformanceTests : ToolProviderConformanceTests +{ + protected override object Provider { get; } = new SearchTools(); +} From 2c783a6acd10c4718f91611529613c4ac526ee3a Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Sun, 18 Oct 2026 23:42:30 -0400 Subject: [PATCH 2/5] Cap indexed text and read every candidate for a filtered search --- src/FluentCMS.Core/Search/SearchText.cs | 17 ++++++++++++++++- .../20261019033140_SearchDocuments.cs | 8 ++++---- .../Search/EntrySearch.cs | 5 ++++- tests/FluentCMS.Tests/Search/SearchTextTests.cs | 11 +++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/FluentCMS.Core/Search/SearchText.cs b/src/FluentCMS.Core/Search/SearchText.cs index bf56afb69..5d57d1352 100644 --- a/src/FluentCMS.Core/Search/SearchText.cs +++ b/src/FluentCMS.Core/Search/SearchText.cs @@ -10,11 +10,26 @@ namespace FluentCMS.Core.Search; // "19.99" or "A-4411" match the same way on PostgreSQL and SQLite. The migrations that index existing entries do the same. public static class SearchText { + // Well inside PostgreSQL's 1 MB limit on a tsvector, so even a very long entry is stored; words past it are not searchable. + public const int MaxLength = 100_000; + public static string From(JsonElement fields) { var text = new StringBuilder(); Append(fields, text); - return string.Join(' ', SearchTerms.Words(text.ToString())); + + var words = new StringBuilder(); + foreach (var word in SearchTerms.Words(text.ToString())) + { + if (words.Length + word.Length + 1 > MaxLength) + { + break; + } + + words.Append(words.Length == 0 ? string.Empty : " ").Append(word); + } + + return words.ToString(); } private static void Append(JsonElement value, StringBuilder text) diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs index 843d165fc..c2ddab329 100644 --- a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations; #nullable disable @@ -22,14 +22,14 @@ protected override void Up(MigrationBuilder migrationBuilder) """); // Every entry that already exists, flattened as SearchText does: each string and number of its latest version, as - // lowercase words. + // lowercase words, cut at SearchText.MaxLength. migrationBuilder.Sql(""" INSERT INTO "SearchDocuments" ("EntryId", "Text") SELECT entry."Id", - trim(regexp_replace(lower(coalesce(( + left(trim(regexp_replace(lower(coalesce(( SELECT string_agg(item #>> '{}', ' ') FROM jsonb_path_query(version."Fields", 'strict $.**') AS item - WHERE jsonb_typeof(item) IN ('string', 'number')), '')), '[^[:alnum:]]+', ' ', 'g')) + WHERE jsonb_typeof(item) IN ('string', 'number')), '')), '[^[:alnum:]]+', ' ', 'g')), 100000) FROM "Entries" AS entry JOIN "EntryVersions" AS version ON version."EntryId" = entry."Id" AND version."Version" = entry."LatestVersion"; """); diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs index bf1ec39e8..62bde1531 100644 --- a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -29,7 +29,10 @@ public async Task> SearchAsync(SearchQuery query, Cancella } var readable = await permissions.ReadableTypesAsync(cancellationToken); - var matches = await index.MatchAsync(terms, Math.Min(query.Limit * CandidatesPerHit, MaxCandidates), cancellationToken); + // A filter can drop most matches, so a filtered search reads as many as it may. + var filtered = query.ContentTypes is not null || query.States is not null || query.PublishedOnly; + var candidates = filtered ? MaxCandidates : Math.Min(query.Limit * CandidatesPerHit, MaxCandidates); + var matches = await index.MatchAsync(terms, candidates, cancellationToken); var ranks = matches.ToDictionary(match => match.EntryId, match => match.Rank); var matchedIds = ranks.Keys.ToList(); diff --git a/tests/FluentCMS.Tests/Search/SearchTextTests.cs b/tests/FluentCMS.Tests/Search/SearchTextTests.cs index 1e890dfb3..ef0f56ecb 100644 --- a/tests/FluentCMS.Tests/Search/SearchTextTests.cs +++ b/tests/FluentCMS.Tests/Search/SearchTextTests.cs @@ -21,6 +21,17 @@ public void An_entrys_text_is_every_string_and_number_in_its_fields_as_lowercase Assert.Equal("widget pro a 4411 url https example com spec 19 99 garden café", SearchText.From(fields)); } + [Fact] + public void A_very_long_entry_is_cut_at_a_word_within_the_limit() + { + var fields = JsonSerializer.SerializeToElement(new { body = string.Join(' ', Enumerable.Repeat("lorem", 30_000)) }); + + var text = SearchText.From(fields); + + Assert.True(text.Length <= SearchText.MaxLength); + Assert.EndsWith("lorem", text); + } + [Theory] [InlineData("4411", new[] { "4411" })] [InlineData(" Widget A-4411 widget ", new[] { "widget", "a", "4411" })] From 1d14122897155167f0074c626780c0b9d0cb78ef Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Mon, 19 Oct 2026 00:46:36 -0400 Subject: [PATCH 3/5] Show the in-review state in search, read every candidate, and refuse empty filters --- CONTRIBUTING.md | 2 +- .../Builtin/Tools/SearchTools.cs | 3 ++ src/FluentCMS.Core/Search/ISearch.cs | 1 + .../Search/EntrySearch.cs | 52 +++++++++++++------ .../Mcp/ConnectedAgent.cs | 17 ++++++ .../Mcp/SearchToolsTests.cs | 35 ++++++++++++- .../Search/SearchTests.cs | 51 ++++++++++++++++-- 7 files changed, 139 insertions(+), 22 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa959432f..079f76a78 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -135,7 +135,7 @@ An LLM-judged type asks a model only through `IModelGateway`, which it reads fro A check is a versioned definition under a slug: a check type and its settings, a description of what it enforces, a severity (block or warn), an optional scope of content types and fields, and test examples, at least one it must pass and one it must fail. `ICheckDirectory` activates a version only after running its examples, and names each example that does not get its verdict (CHK-02). A revision leaves the active version in place until the new one is activated, and only a person activates or deactivates a check. `ICheckRunner` runs the active checks that apply to a piece of content and gives each one's status: pass, warn for a failing check of warn severity, fail, or needs review. Agents run it on a draft with the `check_draft` tool before they propose it (CHK-04). -Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. +Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads up to 1,000 matches before it drops what the scope, deletions and filters leave out, and gives each entry the state the entry tools show, in review included. ## Code style diff --git a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs index e54fdd21d..4deaf2759 100644 --- a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs +++ b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs @@ -81,6 +81,9 @@ private static async Task SearchAsync(ToolInvocation call, Cancellat ["publishedVersion"] = hit.PublishedVersion, ["via"] = hit.Via?.ToString(), })]), + ["next"] = found.Hits.Count == 0 + ? "Nothing matched. Search for fewer or shorter words, or leave out a filter." + : "Read a hit with get_entry, or an entity's facts with get_entity_facts. A hit with 'via' references that match.", }); } } diff --git a/src/FluentCMS.Core/Search/ISearch.cs b/src/FluentCMS.Core/Search/ISearch.cs index 2abc1bc6d..20c3c6d3b 100644 --- a/src/FluentCMS.Core/Search/ISearch.cs +++ b/src/FluentCMS.Core/Search/ISearch.cs @@ -43,4 +43,5 @@ public static class SearchErrorCodes public const string QueryEmpty = "search_query_empty"; public const string QueryTooLong = "search_query_too_long"; public const string LimitInvalid = "search_limit_invalid"; + public const string FilterEmpty = "search_filter_empty"; } diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs index 62bde1531..c88ad5524 100644 --- a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -13,8 +13,7 @@ namespace FluentCMS.Infrastructure.Search; // the index. A match is followed to the entries whose latest version references it, such as the page that shows a matching entity. internal sealed class EntrySearch(FluentCmsDbContext context, ISearchIndex index, IAgentPermissions permissions) : ISearch { - // Matches read before filtering, so filters that drop many still leave enough to fill the limit. - private const int CandidatesPerHit = 10; + // Matches read before the scope, deletions and filters drop some, so the limit still fills when they drop many. private const int MaxCandidates = 1000; public async Task> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default) @@ -29,10 +28,7 @@ public async Task> SearchAsync(SearchQuery query, Cancella } var readable = await permissions.ReadableTypesAsync(cancellationToken); - // A filter can drop most matches, so a filtered search reads as many as it may. - var filtered = query.ContentTypes is not null || query.States is not null || query.PublishedOnly; - var candidates = filtered ? MaxCandidates : Math.Min(query.Limit * CandidatesPerHit, MaxCandidates); - var matches = await index.MatchAsync(terms, candidates, cancellationToken); + var matches = await index.MatchAsync(terms, MaxCandidates, cancellationToken); var ranks = matches.ToDictionary(match => match.EntryId, match => match.Rank); var matchedIds = ranks.Keys.ToList(); @@ -41,7 +37,7 @@ public async Task> SearchAsync(SearchQuery query, Cancella .ToList(); var direct = matched .Where(entry => Passes(entry, query)) - .Select(entry => entry.Hit(ranks[entry.Row.Id], via: null)) + .Select(entry => entry.Hit(ranks[entry.Id], via: null)) .OrderByDescending(hit => hit.Rank) .ThenBy(hit => hit.EntryId) .Take(query.Limit) @@ -54,7 +50,7 @@ public async Task> SearchAsync(SearchQuery query, Cancella } // Only matches the caller may read lead anywhere, so a reference never reveals an entry of a type outside the scope. - var targets = matched.ToDictionary(entry => entry.Row.Id, entry => ranks[entry.Row.Id]); + var targets = matched.ToDictionary(entry => entry.Id, entry => ranks[entry.Id]); var targetIds = targets.Keys.ToList(); var references = await context.EntryReferences .AsNoTracking() @@ -67,12 +63,12 @@ public async Task> SearchAsync(SearchQuery query, Cancella var referencingIds = references.Select(reference => reference.EntryId).Distinct().ToList(); var shown = direct.Select(hit => hit.EntryId).ToHashSet(); var via = (await EntriesAsync(context.Entries.Where(row => referencingIds.Contains(row.Id)), cancellationToken)) - .Where(entry => !shown.Contains(entry.Row.Id) && readable(entry.ContentType) && Passes(entry, query)) + .Where(entry => !shown.Contains(entry.Id) && readable(entry.ContentType) && Passes(entry, query)) .Select(entry => { // The best match the entry references explains why it is shown. var target = references - .Where(reference => reference.EntryId == entry.Row.Id) + .Where(reference => reference.EntryId == entry.Id) .Select(reference => reference.TargetId) .OrderByDescending(id => targets[id]) .ThenBy(id => id) @@ -86,23 +82,39 @@ public async Task> SearchAsync(SearchQuery query, Cancella return Result.Success(new SearchResults([.. direct, .. via])); } + // In review is read from the change sets holding the entry, as the entry tools read it (CNT-01). private async Task> EntriesAsync(IQueryable rows, CancellationToken cancellationToken) { var siteId = Site.Default.Id; + var inReview = EntriesInReview.Ids(context); var found = await rows .AsNoTracking() .Where(row => row.SiteId == siteId && row.DeletedAt == null) .Join(context.ContentTypes, row => row.ContentTypeId, type => type.Id, (row, type) => new { row, type.Slug }) + .Select(pair => new + { + pair.row.Id, + pair.Slug, + pair.row.State, + pair.row.LatestVersion, + pair.row.PublishedVersion, + InReview = inReview.Contains(pair.row.Id), + }) .ToListAsync(cancellationToken); - return [.. found.Select(pair => new Candidate(pair.row, pair.Slug))]; + return [.. found.Select(entry => new Candidate( + entry.Id, + entry.Slug, + entry.InReview ? EntryState.InReview : entry.State, + entry.LatestVersion, + entry.PublishedVersion))]; } private static bool Passes(Candidate entry, SearchQuery query) { return (query.ContentTypes is null || query.ContentTypes.Contains(entry.ContentType, StringComparer.Ordinal)) - && (query.States is null || query.States.Contains(entry.Row.State)) - && (!query.PublishedOnly || entry.Row.PublishedVersion is not null); + && (query.States is null || query.States.Contains(entry.State)) + && (!query.PublishedOnly || entry.PublishedVersion is not null); } private static List Validate(SearchQuery query, IReadOnlyList terms) @@ -120,6 +132,16 @@ private static List Validate(SearchQuery query, IReadOnlyList ter $"A query has at most {SearchTerms.MaxTerms} words of at most {SearchTerms.MaxTermLength} characters each; search for fewer, shorter words.")); } + if (query.ContentTypes is { Count: 0 }) + { + errors.Add(new Error(SearchErrorCodes.FilterEmpty, "The types filter is empty; name at least one content type, or leave it out to search every type you may read.")); + } + + if (query.States is { Count: 0 }) + { + errors.Add(new Error(SearchErrorCodes.FilterEmpty, "The states filter is empty; name at least one state, or leave it out to keep every state.")); + } + if (query.Limit is < 1 or > SearchQuery.MaxLimit) { errors.Add(new Error(SearchErrorCodes.LimitInvalid, $"The limit is from 1 to {SearchQuery.MaxLimit}, and {query.Limit} was given.")); @@ -128,11 +150,11 @@ private static List Validate(SearchQuery query, IReadOnlyList ter return errors; } - private sealed record Candidate(EntryRow Row, string ContentType) + private sealed record Candidate(Guid Id, string ContentType, EntryState State, int LatestVersion, int? PublishedVersion) { public SearchHit Hit(double rank, Guid? via) { - return new SearchHit(Row.Id, ContentType, Row.State, Row.LatestVersion, Row.PublishedVersion, rank, via); + return new SearchHit(Id, ContentType, State, LatestVersion, PublishedVersion, rank, via); } } } diff --git a/tests/FluentCMS.IntegrationTests/Mcp/ConnectedAgent.cs b/tests/FluentCMS.IntegrationTests/Mcp/ConnectedAgent.cs index 6c322813d..0b65e632e 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/ConnectedAgent.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/ConnectedAgent.cs @@ -3,6 +3,7 @@ using FluentCMS.Core.Schema; using FluentCMS.Core.Sites; using FluentCMS.Infrastructure.Persistence; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using ModelContextProtocol.Client; using ModelContextProtocol.Protocol; @@ -60,6 +61,22 @@ public async Task AsPersonAsync(Func>(administrator, action)).Value; } + // Runs as the agent, as a call over MCP would, for work a test drives without a tool. + public async Task AsAgentAsync(Func> action) + { + var accessor = host.App.Services.GetRequiredService(); + await using var scope = host.App.Services.CreateAsyncScope(); + accessor.HttpContext = new DefaultHttpContext { RequestServices = scope.ServiceProvider, User = AgentPrincipal.For(Credential, "test") }; + try + { + return await action(scope.ServiceProvider); + } + finally + { + accessor.HttpContext = null; + } + } + public Task CreateAsPersonAsync(string slug) { return AsPersonAsync(directory => diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs index 77a8288fd..a86e20ca8 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -3,6 +3,7 @@ using FluentCMS.Core.Search; using FluentCMS.Core.Tools; using FluentCMS.Infrastructure.Persistence; +using Microsoft.Extensions.DependencyInjection; namespace FluentCMS.IntegrationTests.Mcp; @@ -21,21 +22,53 @@ public async Task An_agent_finds_entries_of_the_types_it_may_read_and_no_others( var found = await agent.CallAsync("search_entries", new() { ["query"] = "4411" }); var drafts = await agent.CallAsync("search_entries", new() { ["query"] = "4411", ["states"] = new[] { "draft" }, ["limit"] = 5 }); + var published = await agent.CallAsync("search_entries", new() { ["query"] = "4411", ["states"] = new[] { "published" } }); + var service = await agent.AsAgentAsync(services => services.GetRequiredService().SearchAsync(new SearchQuery("4411"))); var hit = Assert.Single(found["hits"]!.AsArray())!; Assert.Equal((page.ToString(), "page", "draft", (string?)null), ((string?)hit["id"], (string?)hit["contentType"], (string?)hit["state"], (string?)hit["via"])); + Assert.Equal([page], service.Value.Hits.Select(serviceHit => serviceHit.EntryId)); Assert.Single(drafts["hits"]!.AsArray()); + Assert.Empty(published["hits"]!.AsArray()); + Assert.Contains("get_entry", (string?)found["next"]); + Assert.Contains("Nothing matched", (string?)published["next"]); + } + + // Matches the agent may not read are dropped before the limit, so better-ranked ones of other types do not crowd out its own. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task Matches_outside_the_scope_do_not_crowd_out_the_ones_the_agent_may_read(string provider) + { + await using var agent = await ConnectedAgent.StartAsync(provider, types: ["page"]); + await agent.CreateAsPersonAsync("page"); + await agent.CreateAsPersonAsync("product"); + foreach (var _ in Enumerable.Range(0, 15)) + { + await CreateAsync(agent, "product", "4411 4411 4411"); + } + + var page = await CreateAsync(agent, "page", "Spring sale on part 4411, while the stock lasts, in every store"); + + var found = await agent.CallAsync("search_entries", new() { ["query"] = "4411", ["limit"] = 1 }); + + Assert.Equal(page.ToString(), (string?)Assert.Single(found["hits"]!.AsArray())!["id"]); } [Fact] - public async Task A_query_without_words_or_an_unknown_state_is_a_tool_error() + public async Task A_query_without_words_an_unknown_state_or_an_empty_filter_is_a_tool_error() { await using var agent = await ConnectedAgent.StartAsync(DatabaseProviders.Sqlite); var empty = await agent.FailAsync("search_entries", new() { ["query"] = "?!" }); var state = await agent.FailAsync("search_entries", new() { ["query"] = "4411", ["states"] = new[] { "live" } }); + var noTypes = await agent.FailAsync("search_entries", new() { ["query"] = "4411", ["types"] = Array.Empty() }); + var noStates = await agent.FailAsync("search_entries", new() { ["query"] = "4411", ["states"] = Array.Empty() }); Assert.Equal(SearchErrorCodes.QueryEmpty, (string?)empty["errors"]![0]!["code"]); + Assert.Equal(SearchErrorCodes.FilterEmpty, (string?)noTypes["errors"]![0]!["code"]); + Assert.Contains("types filter is empty", (string?)noTypes["errors"]![0]!["message"]); + Assert.Equal(SearchErrorCodes.FilterEmpty, (string?)noStates["errors"]![0]!["code"]); Assert.Equal(ToolErrorCodes.ArgumentInvalid, (string?)state["errors"]![0]!["code"]); Assert.Contains("'live' is not an entry state", (string?)state["errors"]![0]!["message"]); } diff --git a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs index c6ef33414..fd56c9cfe 100644 --- a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs +++ b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs @@ -1,8 +1,10 @@ +using FluentCMS.Core.ChangeSets; using FluentCMS.Core.Content; using FluentCMS.Core.Persistence; using FluentCMS.Core.Results; using FluentCMS.Core.Search; using FluentCMS.Infrastructure.Persistence; +using FluentCMS.IntegrationTests.ChangeSets; using FluentCMS.IntegrationTests.Content; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -41,6 +43,31 @@ public async Task A_sku_finds_the_entity_and_the_page_that_references_it_and_fil Assert.Equal([product], publishedAfter.Select(hit => hit.EntryId)); } + // In review is read from the change sets, as the entry tools read it, so a hit's state and the filter agree with get_entry. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task The_state_filter_keeps_entries_by_the_state_the_entry_tools_show_in_review_included(string provider) + { + await using var scenario = await EntryChangeSetTests.ScenarioAsync(provider); + var (page, product) = await EntryChangeSetTests.CatalogAsync(scenario); + await scenario.PublishAsync(EntryChangeSetTests.Operation(OperationKind.Create, ItemKind.Entity, product, null, 1)); + var review = await scenario.OpenAsync(EntryChangeSetTests.Operation(OperationKind.Create, ItemKind.Entry, page, null, 1)); + Assert.True((await scenario.RunAsync(directory => directory.TransitionAsync(review.Id, ChangeSetState.ChecksRunning))).IsSuccess); + + var all = await SearchAsync(scenario.Services, new SearchQuery("widget")); + var published = await SearchAsync(scenario.Services, new SearchQuery("widget") { States = [EntryState.Published] }); + var inReview = await SearchAsync(scenario.Services, new SearchQuery("widget") { States = [EntryState.InReview] }); + var drafts = await SearchAsync(scenario.Services, new SearchQuery("widget") { States = [EntryState.Draft] }); + + Assert.Equal( + [(page, EntryState.InReview), (product, EntryState.Published)], + all.Select(hit => (hit.EntryId, hit.State)).OrderBy(hit => hit.State == EntryState.Published)); + Assert.Equal([product], published.Select(hit => hit.EntryId)); + Assert.Equal([page], inReview.Select(hit => hit.EntryId)); + Assert.Empty(drafts); + } + [Theory] [InlineData(DatabaseProviders.Sqlite)] [PostgresData] @@ -100,15 +127,19 @@ public async Task Entries_that_existed_before_the_index_are_found_after_the_migr [Theory] [InlineData(DatabaseProviders.Sqlite)] [PostgresData] - public async Task A_query_without_words_or_with_too_many_or_a_limit_out_of_range_is_refused(string provider) + public async Task A_query_without_words_with_too_many_an_empty_filter_or_a_limit_out_of_range_is_refused(string provider) { await using var scenario = await EntryScenario.CreateAsync(provider); var empty = await RunAsync(scenario, new SearchQuery(" -- ")); var tooMany = await RunAsync(scenario, new SearchQuery(string.Join(' ', Enumerable.Range(1, SearchTerms.MaxTerms + 1).Select(word => $"w{word}")))); var limit = await RunAsync(scenario, new SearchQuery("widget") { Limit = 0 }); + var noTypes = await RunAsync(scenario, new SearchQuery("widget") { ContentTypes = [] }); + var noStates = await RunAsync(scenario, new SearchQuery("widget") { States = [] }); Assert.Equal(SearchErrorCodes.QueryEmpty, Assert.Single(empty.Errors).Code); + Assert.Equal(SearchErrorCodes.FilterEmpty, Assert.Single(noTypes.Errors).Code); + Assert.Equal(SearchErrorCodes.FilterEmpty, Assert.Single(noStates.Errors).Code); Assert.Equal(SearchErrorCodes.QueryTooLong, Assert.Single(tooMany.Errors).Code); Assert.Equal(SearchErrorCodes.LimitInvalid, Assert.Single(limit.Errors).Code); } @@ -126,16 +157,26 @@ private static async Task CatalogAsync(string provider) return new Catalog(entries, product, page, other); } - private static async Task> SearchAsync(EntryScenario scenario, SearchQuery query) + private static Task> SearchAsync(EntryScenario scenario, SearchQuery query) { - var results = await RunAsync(scenario, query); + return SearchAsync(scenario.Services, query); + } + + private static async Task> SearchAsync(IServiceProvider services, SearchQuery query) + { + var results = await RunAsync(services, query); Assert.True(results.IsSuccess, string.Join("; ", results.Errors.Select(error => error.Message))); return results.Value.Hits; } - private static async Task> RunAsync(EntryScenario scenario, SearchQuery query) + private static Task> RunAsync(EntryScenario scenario, SearchQuery query) + { + return RunAsync(scenario.Services, query); + } + + private static async Task> RunAsync(IServiceProvider services, SearchQuery query) { - await using var scope = scenario.Services.CreateAsyncScope(); + await using var scope = services.CreateAsyncScope(); return await scope.ServiceProvider.GetRequiredService().SearchAsync(query); } From 98889807b6bd422bb6fad21e8f755c9a51f23095 Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Mon, 19 Oct 2026 01:55:20 -0400 Subject: [PATCH 4/5] Page through search matches, leave GUIDs out of the index, and re-scaffold the migrations on v2 --- CONTRIBUTING.md | 2 +- .../Builtin/Tools/SearchTools.cs | 9 +- src/FluentCMS.Core/Search/ISearch.cs | 4 +- src/FluentCMS.Core/Search/ISearchIndex.cs | 5 +- src/FluentCMS.Core/Search/SearchText.cs | 5 +- ...frastructureServiceCollectionExtensions.cs | 7 +- ...0261019054640_SearchDocuments.Designer.cs} | 122 +++++++++++++++++- ...s.cs => 20261019054640_SearchDocuments.cs} | 9 +- ...0261019054608_SearchDocuments.Designer.cs} | 122 +++++++++++++++++- ...s.cs => 20261019054608_SearchDocuments.cs} | 10 +- .../Search/EntrySearch.cs | 51 ++++++-- .../Search/SearchIndexes.cs | 8 +- .../Search/SearchOptions.cs | 25 ++++ .../Mcp/SearchToolsTests.cs | 1 + .../Search/SearchTests.cs | 60 +++++++++ .../FluentCMS.Tests/Search/SearchTextTests.cs | 11 ++ 16 files changed, 416 insertions(+), 35 deletions(-) rename src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/{20261019033140_SearchDocuments.Designer.cs => 20261019054640_SearchDocuments.Designer.cs} (90%) rename src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/{20261019033140_SearchDocuments.cs => 20261019054640_SearchDocuments.cs} (81%) rename src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/{20261019033130_SearchDocuments.Designer.cs => 20261019054608_SearchDocuments.Designer.cs} (90%) rename src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/{20261019033130_SearchDocuments.cs => 20261019054608_SearchDocuments.cs} (78%) create mode 100644 src/FluentCMS.Infrastructure/Search/SearchOptions.cs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f8af6f1a..9dbba138e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -136,7 +136,7 @@ An LLM-judged type asks a model only through `IModelGateway`, which it reads fro A check is a versioned definition under a slug: a check type and its settings, a description of what it enforces, a severity (block or warn), an optional scope of content types and fields, and test examples, at least one it must pass and one it must fail. `ICheckDirectory` activates a version only after running its examples, and names each example that does not get its verdict (CHK-02). A revision leaves the active version in place until the new one is activated, and only a person activates or deactivates a check. `ICheckRunner` runs the active checks that apply to a piece of content and gives each one's status: pass, warn for a failing check of warn severity, fail, or needs review. Agents run it on a draft with the `check_draft` tool before they propose it (CHK-04). -Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads up to 1,000 matches before it drops what the scope, deletions and filters leave out, and gives each entry the state the entry tools show, in review included. +Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads the matches a page at a time, `Search__PageSize` (1,000 by default), and drops what the scope, deletions and filters leave out, until enough are left, the index has no more, or it has read `Search__MaxMatches` (10,000). A result that stopped there says it is not complete. Each hit has the state the entry tools show, in review included. A string that is a GUID, such as a reference, is not indexed. ## Code style diff --git a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs index 4deaf2759..d7fac739d 100644 --- a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs +++ b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs @@ -81,9 +81,12 @@ private static async Task SearchAsync(ToolInvocation call, Cancellat ["publishedVersion"] = hit.PublishedVersion, ["via"] = hit.Via?.ToString(), })]), - ["next"] = found.Hits.Count == 0 - ? "Nothing matched. Search for fewer or shorter words, or leave out a filter." - : "Read a hit with get_entry, or an entity's facts with get_entity_facts. A hit with 'via' references that match.", + ["complete"] = found.Complete, + ["next"] = !found.Complete + ? "The search stopped after reading the most matches it reads, so hits may be missing. Add a word to narrow it." + : found.Hits.Count == 0 + ? "Nothing matched. Search for fewer or shorter words, or leave out a filter." + : "Read a hit with get_entry, or an entity's facts with get_entity_facts. A hit with 'via' references that match.", }); } } diff --git a/src/FluentCMS.Core/Search/ISearch.cs b/src/FluentCMS.Core/Search/ISearch.cs index 20c3c6d3b..e9b6b1a6d 100644 --- a/src/FluentCMS.Core/Search/ISearch.cs +++ b/src/FluentCMS.Core/Search/ISearch.cs @@ -26,7 +26,9 @@ public sealed record SearchQuery(string Text) public int Limit { get; init; } = DefaultLimit; } -public sealed record SearchResults(IReadOnlyList Hits); +// Complete is false when the search stopped at the most matches it reads, so better-ranked matches outside the filters or the +// scope may have crowded out some that would pass them. +public sealed record SearchResults(IReadOnlyList Hits, bool Complete = true); // A direct match has no Via; an entry that references a match names it in Via, such as the page that shows a matching entity. public sealed record SearchHit( diff --git a/src/FluentCMS.Core/Search/ISearchIndex.cs b/src/FluentCMS.Core/Search/ISearchIndex.cs index 211731c15..7fef1e8fd 100644 --- a/src/FluentCMS.Core/Search/ISearchIndex.cs +++ b/src/FluentCMS.Core/Search/ISearchIndex.cs @@ -11,8 +11,9 @@ public interface ISearchIndex // Replaces what the index holds for the entry. Task IndexAsync(SearchDocument document, CancellationToken cancellationToken = default); - // Entries whose text has every term, each matched as a prefix, best first. Terms are words of letters and digits (SearchTerms). - Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default); + // Entries whose text has every term, each matched as a prefix, best first and in the same order on every call, so a caller pages + // through them by skipping those it has read. Terms are words of letters and digits (SearchTerms). + Task> MatchAsync(IReadOnlyList terms, int skip, int limit, CancellationToken cancellationToken = default); } public sealed record SearchDocument(Guid EntryId, string Text); diff --git a/src/FluentCMS.Core/Search/SearchText.cs b/src/FluentCMS.Core/Search/SearchText.cs index 5d57d1352..293ab7df4 100644 --- a/src/FluentCMS.Core/Search/SearchText.cs +++ b/src/FluentCMS.Core/Search/SearchText.cs @@ -6,7 +6,8 @@ namespace FluentCMS.Core.Search; // What an entry is found by: every string and number in its fields, a fact's value and source included, in field order, as the -// lowercase words SearchTerms splits a query into. Indexing words rather than raw text keeps both engines' tokenizers out of it, so +// lowercase words SearchTerms splits a query into. A string that is a GUID, such as a reference or a media asset's id, is left out: +// its parts would match queries such as "01". Indexing words rather than raw text keeps both engines' tokenizers out of it, so // "19.99" or "A-4411" match the same way on PostgreSQL and SQLite. The migrations that index existing entries do the same. public static class SearchText { @@ -50,7 +51,7 @@ private static void Append(JsonElement value, StringBuilder text) } break; - case JsonValueKind.String: + case JsonValueKind.String when !Guid.TryParseExact(value.GetString(), "D", out _): text.Append(value.GetString()).Append(' '); break; case JsonValueKind.Number: diff --git a/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs b/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs index d63b4bc19..2f0851e76 100644 --- a/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs +++ b/src/FluentCMS.Infrastructure/InfrastructureServiceCollectionExtensions.cs @@ -42,7 +42,7 @@ public static IServiceCollection AddFluentCmsInfrastructure(this IServiceCollect var database = configuration.GetSection(DatabaseOptions.SectionName).Get() ?? new DatabaseOptions(); AddDatabase(services, database); - AddSearch(services, database.ResolveProvider()); + AddSearch(services, configuration, database.ResolveProvider()); AddIdentity(services); AddAudit(services); AddSchema(services); @@ -77,8 +77,11 @@ private static void AddDatabase(IServiceCollection services, DatabaseOptions dat } // The index of the configured provider, scoped because it writes through the request's context (ADR-0010). - private static void AddSearch(IServiceCollection services, string provider) + private static void AddSearch(IServiceCollection services, IConfiguration configuration, string provider) { + services.AddOptions().Bind(configuration.GetSection(SearchOptions.SectionName)).ValidateOnStart(); + services.AddSingleton, SearchOptionsValidation>(); + if (provider == DatabaseProviders.PostgreSql) { services.AddFluentCmsExtension(ServiceLifetime.Scoped); diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.Designer.cs similarity index 90% rename from src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs rename to src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.Designer.cs index 649917357..8a6abc68b 100644 --- a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.Designer.cs +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.Designer.cs @@ -13,7 +13,7 @@ namespace FluentCMS.Infrastructure.Persistence.Migrations.PostgreSql { [DbContext(typeof(PostgreSqlFluentCmsDbContext))] - [Migration("20261019033140_SearchDocuments")] + [Migration("20261019054640_SearchDocuments")] partial class SearchDocuments { /// @@ -639,6 +639,108 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("LatestVersion") + .HasColumnType("integer"); + + b.Property("PublishedVersion") + .HasColumnType("integer"); + + b.Property("SiteId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SiteId"); + + b.ToTable("MediaAssets", null, t => + { + t.HasCheckConstraint("CK_MediaAssets_LatestVersion", "\"LatestVersion\" >= 1"); + + t.HasCheckConstraint("CK_MediaAssets_PublishedVersion", "\"PublishedVersion\" IS NULL OR (\"PublishedVersion\" >= 1 AND \"PublishedVersion\" <= \"LatestVersion\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetVersionRow", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AltText") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("Caption") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(127) + .HasColumnType("character varying(127)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Folder") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("Length") + .HasColumnType("bigint"); + + b.Property("Sha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Storage") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("StorageKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.PrimitiveCollection>("Tags") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("Version") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Folder"); + + b.HasIndex("AssetId", "Version") + .IsUnique(); + + b.ToTable("MediaAssetVersions", null, t => + { + t.HasTrigger("TR_MediaAssetVersions_NoUpdate"); + + t.HasCheckConstraint("CK_MediaAssetVersions_Length", "\"Length\" > 0"); + + t.HasCheckConstraint("CK_MediaAssetVersions_Version", "\"Version\" >= 1"); + }); + }); + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => { b.Property("Id") @@ -1107,6 +1209,24 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired(); }); + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Media.MediaAssetRow", null) + .WithMany() + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => { b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.cs similarity index 81% rename from src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs rename to src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.cs index c2ddab329..b6e9ed59f 100644 --- a/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019033140_SearchDocuments.cs +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/PostgreSql/20261019054640_SearchDocuments.cs @@ -21,15 +21,18 @@ protected override void Up(MigrationBuilder migrationBuilder) CREATE INDEX "IX_SearchDocuments_Document" ON "SearchDocuments" USING GIN ("Document"); """); - // Every entry that already exists, flattened as SearchText does: each string and number of its latest version, as - // lowercase words, cut at SearchText.MaxLength. + // Every entry that already exists, flattened as SearchText does: each string and number of its latest version but a GUID, + // as lowercase words, cut at SearchText.MaxLength. migrationBuilder.Sql(""" INSERT INTO "SearchDocuments" ("EntryId", "Text") SELECT entry."Id", left(trim(regexp_replace(lower(coalesce(( SELECT string_agg(item #>> '{}', ' ') FROM jsonb_path_query(version."Fields", 'strict $.**') AS item - WHERE jsonb_typeof(item) IN ('string', 'number')), '')), '[^[:alnum:]]+', ' ', 'g')), 100000) + WHERE jsonb_typeof(item) IN ('string', 'number') + AND NOT (jsonb_typeof(item) = 'string' + AND (item #>> '{}') ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')), '')), + '[^[:alnum:]]+', ' ', 'g')), 100000) FROM "Entries" AS entry JOIN "EntryVersions" AS version ON version."EntryId" = entry."Id" AND version."Version" = entry."LatestVersion"; """); diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.Designer.cs similarity index 90% rename from src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs rename to src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.Designer.cs index 2e13c5498..4c825aed0 100644 --- a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.Designer.cs +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.Designer.cs @@ -11,7 +11,7 @@ namespace FluentCMS.Infrastructure.Persistence.Migrations.Sqlite { [DbContext(typeof(SqliteFluentCmsDbContext))] - [Migration("20261019033130_SearchDocuments")] + [Migration("20261019054608_SearchDocuments")] partial class SearchDocuments { /// @@ -633,6 +633,108 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("LatestVersion") + .HasColumnType("INTEGER"); + + b.Property("PublishedVersion") + .HasColumnType("INTEGER"); + + b.Property("SiteId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SiteId"); + + b.ToTable("MediaAssets", null, t => + { + t.HasCheckConstraint("CK_MediaAssets_LatestVersion", "\"LatestVersion\" >= 1"); + + t.HasCheckConstraint("CK_MediaAssets_PublishedVersion", "\"PublishedVersion\" IS NULL OR (\"PublishedVersion\" >= 1 AND \"PublishedVersion\" <= \"LatestVersion\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetVersionRow", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AltText") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("AssetId") + .HasColumnType("TEXT"); + + b.Property("Caption") + .HasMaxLength(2000) + .HasColumnType("TEXT"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(127) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("Folder") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("Length") + .HasColumnType("INTEGER"); + + b.Property("Sha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Storage") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("StorageKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.PrimitiveCollection("Tags") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Version") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("Folder"); + + b.HasIndex("AssetId", "Version") + .IsUnique(); + + b.ToTable("MediaAssetVersions", null, t => + { + t.HasTrigger("TR_MediaAssetVersions_NoUpdate"); + + t.HasCheckConstraint("CK_MediaAssetVersions_Length", "\"Length\" > 0"); + + t.HasCheckConstraint("CK_MediaAssetVersions_Version", "\"Version\" >= 1"); + }); + }); + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => { b.Property("Id") @@ -1097,6 +1199,24 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired(); }); + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Sites.SiteRow", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Infrastructure.Media.MediaAssetVersionRow", b => + { + b.HasOne("FluentCMS.Infrastructure.Media.MediaAssetRow", null) + .WithMany() + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + modelBuilder.Entity("FluentCMS.Infrastructure.Models.ModelCallRow", b => { b.HasOne("FluentCMS.Infrastructure.Agents.AgentCredentialRow", null) diff --git a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.cs similarity index 78% rename from src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs rename to src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.cs index 485de72e4..907ede66d 100644 --- a/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019033130_SearchDocuments.cs +++ b/src/FluentCMS.Infrastructure/Persistence/Migrations/Sqlite/20261019054608_SearchDocuments.cs @@ -7,6 +7,10 @@ namespace FluentCMS.Infrastructure.Persistence.Migrations.Sqlite /// public partial class SearchDocuments : Migration { + // A GUID in hyphenated form, which SearchText leaves out. + private static readonly string GuidPattern = + string.Join("-", new[] { 8, 4, 4, 4, 12 }.Select(length => string.Concat(Enumerable.Repeat("[0-9a-fA-F]", length)))); + /// protected override void Up(MigrationBuilder migrationBuilder) { @@ -16,15 +20,15 @@ protected override void Up(MigrationBuilder migrationBuilder) CREATE VIRTUAL TABLE "SearchDocuments" USING fts5("EntryId" UNINDEXED, "Text", tokenize = 'unicode61 remove_diacritics 0'); """); - // Every entry that already exists, flattened as SearchText does: each string and number of its latest version. The + // Every entry that already exists, flattened as SearchText does: each string and number of its latest version but a GUID. The // tokenizer splits on everything but letters and digits and folds case, as SearchText does before indexing. - migrationBuilder.Sql(""" + migrationBuilder.Sql($""" INSERT INTO "SearchDocuments" ("EntryId", "Text") SELECT entry."Id", coalesce(( SELECT group_concat(item.value, ' ') FROM json_tree(version."Fields") AS item - WHERE item.type IN ('text', 'integer', 'real')), '') + WHERE item.type IN ('text', 'integer', 'real') AND NOT (item.type = 'text' AND item.value GLOB '{GuidPattern}')), '') FROM "Entries" AS entry JOIN "EntryVersions" AS version ON version."EntryId" = entry."Id" AND version."Version" = entry."LatestVersion"; """); diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs index c88ad5524..055673af7 100644 --- a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -6,15 +6,15 @@ using FluentCMS.Infrastructure.Content; using FluentCMS.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; namespace FluentCMS.Infrastructure.Search; // The index finds the text; the entry rows decide the rest, so deleting, restoring and publishing change results without touching // the index. A match is followed to the entries whose latest version references it, such as the page that shows a matching entity. -internal sealed class EntrySearch(FluentCmsDbContext context, ISearchIndex index, IAgentPermissions permissions) : ISearch +internal sealed class EntrySearch(FluentCmsDbContext context, ISearchIndex index, IAgentPermissions permissions, IOptions options) : ISearch { - // Matches read before the scope, deletions and filters drop some, so the limit still fills when they drop many. - private const int MaxCandidates = 1000; + private readonly SearchOptions _settings = options.Value; public async Task> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default) { @@ -28,13 +28,7 @@ public async Task> SearchAsync(SearchQuery query, Cancella } var readable = await permissions.ReadableTypesAsync(cancellationToken); - var matches = await index.MatchAsync(terms, MaxCandidates, cancellationToken); - var ranks = matches.ToDictionary(match => match.EntryId, match => match.Rank); - var matchedIds = ranks.Keys.ToList(); - - var matched = (await EntriesAsync(context.Entries.Where(row => matchedIds.Contains(row.Id)), cancellationToken)) - .Where(entry => readable(entry.ContentType)) - .ToList(); + var (matched, ranks, complete) = await MatchedAsync(terms, readable, query, cancellationToken); var direct = matched .Where(entry => Passes(entry, query)) .Select(entry => entry.Hit(ranks[entry.Id], via: null)) @@ -46,7 +40,7 @@ public async Task> SearchAsync(SearchQuery query, Cancella var room = query.Limit - direct.Count; if (room == 0) { - return Result.Success(new SearchResults(direct)); + return Result.Success(new SearchResults(direct, complete)); } // Only matches the caller may read lead anywhere, so a reference never reveals an entry of a type outside the scope. @@ -79,7 +73,40 @@ public async Task> SearchAsync(SearchQuery query, Cancella .ThenBy(hit => hit.EntryId) .Take(room); - return Result.Success(new SearchResults([.. direct, .. via])); + return Result.Success(new SearchResults([.. direct, .. via], complete)); + } + + // The readable matches, a page at a time in rank order. Better-ranked matches the scope, deletions or filters drop would otherwise + // crowd out the rest, so reading stops only once enough pass the filters, the index has no more, or MaxMatches are read. + private async Task<(List Matched, Dictionary Ranks, bool Complete)> MatchedAsync( + IReadOnlyList terms, + Func readable, + SearchQuery query, + CancellationToken cancellationToken) + { + var matched = new List(); + var ranks = new Dictionary(); + var read = 0; + + while (read < _settings.MaxMatches) + { + var take = Math.Min(_settings.PageSize, _settings.MaxMatches - read); + var page = await index.MatchAsync(terms, read, take, cancellationToken); + read += page.Count; + + var fresh = page.Where(match => ranks.TryAdd(match.EntryId, match.Rank)).Select(match => match.EntryId).ToList(); + matched.AddRange((await EntriesAsync(context.Entries.Where(row => fresh.Contains(row.Id)), cancellationToken)) + .Where(entry => readable(entry.ContentType))); + + if (page.Count < take || matched.Count(entry => Passes(entry, query)) >= query.Limit) + { + return (matched, ranks, true); + } + } + + // Whether anything is left past the last match read. + var more = await index.MatchAsync(terms, read, 1, cancellationToken); + return (matched, ranks, more.Count == 0); } // In review is read from the change sets holding the entry, as the entry tools read it (CNT-01). diff --git a/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs b/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs index 3d03ce420..18851b942 100644 --- a/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs +++ b/src/FluentCMS.Infrastructure/Search/SearchIndexes.cs @@ -21,7 +21,7 @@ ON CONFLICT ("EntryId") DO UPDATE SET "Text" = EXCLUDED."Text" cancellationToken); } - public async Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default) + public async Task> MatchAsync(IReadOnlyList terms, int skip, int limit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(terms); @@ -34,7 +34,7 @@ public async Task> MatchAsync(IReadOnlyList t FROM "SearchDocuments", to_tsquery('simple', {query}) AS matched WHERE "Document" @@ matched ORDER BY "Rank" DESC, "EntryId" - LIMIT {limit} + LIMIT {limit} OFFSET {skip} """) .ToListAsync(cancellationToken); @@ -55,7 +55,7 @@ await context.Database.ExecuteSqlAsync( cancellationToken); } - public async Task> MatchAsync(IReadOnlyList terms, int limit, CancellationToken cancellationToken = default) + public async Task> MatchAsync(IReadOnlyList terms, int skip, int limit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(terms); @@ -68,7 +68,7 @@ public async Task> MatchAsync(IReadOnlyList t FROM "SearchDocuments" WHERE "SearchDocuments" MATCH {query} ORDER BY bm25("SearchDocuments"), "EntryId" - LIMIT {limit} + LIMIT {limit} OFFSET {skip} """) .ToListAsync(cancellationToken); diff --git a/src/FluentCMS.Infrastructure/Search/SearchOptions.cs b/src/FluentCMS.Infrastructure/Search/SearchOptions.cs new file mode 100644 index 000000000..301897ee6 --- /dev/null +++ b/src/FluentCMS.Infrastructure/Search/SearchOptions.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; + +namespace FluentCMS.Infrastructure.Search; + +// How many matches a search reads. The scope, deletions and filters are applied to the matches, so a search reads them a page at a +// time until it has enough hits, and stops at MaxMatches, which covers every entry at M1's 10,000 (SRC-01). +public sealed class SearchOptions +{ + public const string SectionName = "Search"; + + public int PageSize { get; set; } = 1_000; + + public int MaxMatches { get; set; } = 10_000; +} + +internal sealed class SearchOptionsValidation : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, SearchOptions options) + { + return options.PageSize >= 1 && options.MaxMatches >= options.PageSize + ? ValidateOptionsResult.Success + : ValidateOptionsResult.Fail( + $"Search__PageSize is {options.PageSize} and Search__MaxMatches is {options.MaxMatches}; the page size must be 1 or more and at most the maximum."); + } +} diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs index a86e20ca8..9d2d70e44 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -30,6 +30,7 @@ public async Task An_agent_finds_entries_of_the_types_it_may_read_and_no_others( Assert.Equal([page], service.Value.Hits.Select(serviceHit => serviceHit.EntryId)); Assert.Single(drafts["hits"]!.AsArray()); Assert.Empty(published["hits"]!.AsArray()); + Assert.True((bool)found["complete"]!); Assert.Contains("get_entry", (string?)found["next"]); Assert.Contains("Nothing matched", (string?)published["next"]); } diff --git a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs index fd56c9cfe..c21a0c881 100644 --- a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs +++ b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs @@ -4,6 +4,7 @@ using FluentCMS.Core.Results; using FluentCMS.Core.Search; using FluentCMS.Infrastructure.Persistence; +using FluentCMS.Infrastructure.Search; using FluentCMS.IntegrationTests.ChangeSets; using FluentCMS.IntegrationTests.Content; using Microsoft.EntityFrameworkCore; @@ -32,6 +33,7 @@ public async Task A_sku_finds_the_entity_and_the_page_that_references_it_and_fil var publishedBefore = await SearchAsync(scenario.Entries, new SearchQuery("4411") { PublishedOnly = true }); await scenario.Entries.Schema.QueryAsync(context => context.Entries.Where(row => row.Id == product).ExecuteUpdateAsync(setters => setters.SetProperty(row => row.PublishedVersion, 1))); var publishedAfter = await SearchAsync(scenario.Entries, new SearchQuery("4411") { PublishedOnly = true }); + var productId = await SearchAsync(scenario.Entries, new SearchQuery(product.ToString()[..8])); Assert.Equal([(product, (Guid?)null), (page, product)], all.Select(hit => (hit.EntryId, hit.Via))); Assert.Equal(("product", EntryState.Draft, 1), (all[0].ContentType, all[0].State, all[0].LatestVersion)); @@ -41,6 +43,7 @@ public async Task A_sku_finds_the_entity_and_the_page_that_references_it_and_fil Assert.DoesNotContain(all, hit => hit.EntryId == other); Assert.Empty(publishedBefore); Assert.Equal([product], publishedAfter.Select(hit => hit.EntryId)); + Assert.Empty(productId); } // In review is read from the change sets, as the entry tools read it, so a hit's state and the filter agree with get_entry. @@ -103,6 +106,61 @@ public async Task An_entry_is_found_once_its_write_commits_and_not_after_a_rollb Assert.Equal([note.Id], restored.Select(hit => hit.EntryId)); } + // The index row is written in the write's transaction, so an update that rolls back leaves the old words found and the new ones not. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task An_update_that_rolls_back_leaves_the_index_as_it_was(string provider) + { + await using var scenario = await EntryScenario.CreateAsync(provider); + await scenario.Schema.CreateAsync("note", Field("text", "text")); + var note = (await scenario.CreateAsync("note", """{"text": "first draft borealis"}""")).Value; + + await using (var scope = scenario.Services.CreateAsyncScope()) + { + await using var work = await scope.ServiceProvider.GetRequiredService().BeginAsync(); + var updated = await scope.ServiceProvider.GetRequiredService().UpdateAsync(note.Id, 1, Json("""{"text": "second draft australis"}""")); + Assert.True(updated.IsSuccess); + } + + Assert.Equal([note.Id], (await SearchAsync(scenario, new SearchQuery("borealis"))).Select(hit => hit.EntryId)); + Assert.Empty(await SearchAsync(scenario, new SearchQuery("australis"))); + } + + // Better-ranked matches a filter drops are read past a page at a time, up to the most a search reads, and a search that stops + // there says so. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_filtered_search_reads_past_better_ranked_matches_and_says_when_it_stopped_early(string provider) + { + await using var scenario = await EntryScenario.CreateAsync( + provider, + services => services.Configure(search => (search.PageSize, search.MaxMatches) = (20, 60))); + await scenario.Schema.CreateAsync("note", Field("text", "text")); + await scenario.Schema.CreateAsync("page", Field("title", "text")); + foreach (var _ in Enumerable.Range(0, 45)) + { + await scenario.CreateAsync("note", """{"text": "zephyr zephyr zephyr"}"""); + } + + var page = (await scenario.CreateAsync("page", """{"title": "A zephyr in a much longer title of the page with many other words"}""")).Value; + var found = await RunAsync(scenario, new SearchQuery("zephyr") { ContentTypes = ["page"] }); + + foreach (var _ in Enumerable.Range(0, 20)) + { + await scenario.CreateAsync("note", """{"text": "zephyr zephyr zephyr"}"""); + } + + var stopped = await RunAsync(scenario, new SearchQuery("zephyr") { ContentTypes = ["page"] }); + var unfiltered = await RunAsync(scenario, new SearchQuery("zephyr") { Limit = 5 }); + + Assert.Equal([page.Id], found.Value.Hits.Select(hit => hit.EntryId)); + Assert.True(found.Value.Complete); + Assert.Equal((0, false), (stopped.Value.Hits.Count, stopped.Value.Complete)); + Assert.Equal((5, true), (unfiltered.Value.Hits.Count, unfiltered.Value.Complete)); + } + [Theory] [InlineData(DatabaseProviders.Sqlite)] [PostgresData] @@ -120,8 +178,10 @@ public async Task Entries_that_existed_before_the_index_are_found_after_the_migr } var hits = await SearchAsync(scenario.Entries, new SearchQuery("widget 4411")); + var productId = await SearchAsync(scenario.Entries, new SearchQuery(scenario.Product.ToString()[..8])); Assert.Equal([scenario.Product, scenario.Page], hits.Select(hit => hit.EntryId)); + Assert.Empty(productId); } [Theory] diff --git a/tests/FluentCMS.Tests/Search/SearchTextTests.cs b/tests/FluentCMS.Tests/Search/SearchTextTests.cs index ef0f56ecb..03f23c42c 100644 --- a/tests/FluentCMS.Tests/Search/SearchTextTests.cs +++ b/tests/FluentCMS.Tests/Search/SearchTextTests.cs @@ -21,6 +21,17 @@ public void An_entrys_text_is_every_string_and_number_in_its_fields_as_lowercase Assert.Equal("widget pro a 4411 url https example com spec 19 99 garden café", SearchText.From(fields)); } + // An id's parts would match queries such as "01" in every entry that references something. + [Fact] + public void A_string_that_is_a_guid_is_left_out() + { + var fields = JsonDocument.Parse(""" + {"title": "Episode 01", "product": "0199F3A1-7C2B-7D4E-8A1B-2C3D4E5F6A7B", "note": "0199f3a1 is not an id"} + """).RootElement; + + Assert.Equal("episode 01 0199f3a1 is not an id", SearchText.From(fields)); + } + [Fact] public void A_very_long_entry_is_cut_at_a_word_within_the_limit() { From c9a80672868a72787dd14eaf4a6780352bab65bb Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Mon, 19 Oct 2026 02:53:36 -0400 Subject: [PATCH 5/5] Refuse unknown and out-of-scope search types, and test the scope of references --- .../Search/EntrySearch.cs | 39 ++++++++++++++++++ .../Mcp/SearchToolsTests.cs | 40 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs index 055673af7..b206b0e78 100644 --- a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -1,6 +1,7 @@ using FluentCMS.Core.Agents; using FluentCMS.Core.Content; using FluentCMS.Core.Results; +using FluentCMS.Core.Schema; using FluentCMS.Core.Search; using FluentCMS.Core.Sites; using FluentCMS.Infrastructure.Content; @@ -28,6 +29,12 @@ public async Task> SearchAsync(SearchQuery query, Cancella } var readable = await permissions.ReadableTypesAsync(cancellationToken); + var types = await CheckTypesAsync(query.ContentTypes, readable, cancellationToken); + if (types.Count > 0) + { + return Result.Failure(types); + } + var (matched, ranks, complete) = await MatchedAsync(terms, readable, query, cancellationToken); var direct = matched .Where(entry => Passes(entry, query)) @@ -109,6 +116,38 @@ public async Task> SearchAsync(SearchQuery query, Cancella return (matched, ranks, more.Count == 0); } + // Each type a filter names exists and is one the caller may read, as list_entries asks, so a typo or a type outside the scope is + // not taken for a search that matched nothing. + private async Task> CheckTypesAsync(IReadOnlyList? slugs, Func readable, CancellationToken cancellationToken) + { + if (slugs is not { Count: > 0 }) + { + return []; + } + + var siteId = Site.Default.Id; + var known = await context.ContentTypes + .AsNoTracking() + .Where(type => type.SiteId == siteId && slugs.Contains(type.Slug)) + .Select(type => type.Slug) + .ToListAsync(cancellationToken); + + var errors = new List(); + foreach (var slug in slugs.Distinct(StringComparer.Ordinal)) + { + if (!known.Contains(slug, StringComparer.Ordinal)) + { + errors.Add(new Error(SchemaErrorCodes.ContentTypeNotFound, $"There is no content type '{slug}' in this site; list_content_types shows the types there are.")); + } + else if (!readable(slug)) + { + errors.Add(new Error(AgentPermissionErrorCodes.ScopeDenied, $"'{slug}' is not among the content types this credential may read, so it cannot be searched.")); + } + } + + return errors; + } + // In review is read from the change sets holding the entry, as the entry tools read it (CNT-01). private async Task> EntriesAsync(IQueryable rows, CancellationToken cancellationToken) { diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs index 9d2d70e44..acd8d5f0c 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -1,5 +1,7 @@ using System.Text.Json; +using FluentCMS.Core.Agents; using FluentCMS.Core.Content; +using FluentCMS.Core.Schema; using FluentCMS.Core.Search; using FluentCMS.Core.Tools; using FluentCMS.Infrastructure.Persistence; @@ -56,6 +58,44 @@ public async Task Matches_outside_the_scope_do_not_crowd_out_the_ones_the_agent_ Assert.Equal(page.ToString(), (string?)Assert.Single(found["hits"]!.AsArray())!["id"]); } + // Only matches the agent may read lead to the entries that reference them, so a page found through a product would show the + // product's id to an agent that may not read products. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_page_is_not_found_through_a_match_the_agent_may_not_read(string provider) + { + await using var agent = await ConnectedAgent.StartAsync(provider, types: ["page"]); + await agent.AsPersonAsync(directory => + directory.CreateAsync(new NewContentType("product", "product", new ContentTypeDefinition([new FieldDefinition("title", "text")])))); + await agent.AsPersonAsync(directory => + directory.CreateAsync(new NewContentType("page", "page", new ContentTypeDefinition( + [new FieldDefinition("title", "text"), new FieldDefinition("product", "reference", settings: JsonSerializer.SerializeToElement(new { target = "product" }))])))); + var product = await CreateAsync(agent, "product", "Part 4411"); + var page = (await agent.AsPersonAsync(entries => + entries.CreateAsync(new NewEntry("page", JsonSerializer.SerializeToElement(new { title = "Spring sale", product = product.ToString() }))))).Id; + + var found = await agent.CallAsync("search_entries", new() { ["query"] = "4411" }); + var person = await agent.AsPersonAsync(search => search.SearchAsync(new SearchQuery("4411"))); + + Assert.Empty(found["hits"]!.AsArray()); + Assert.Equal([(product, (Guid?)null), (page, product)], person.Hits.Select(hit => (hit.EntryId, hit.Via))); + } + + [Fact] + public async Task A_type_filter_naming_an_unknown_type_or_one_outside_the_scope_is_a_tool_error() + { + await using var agent = await ConnectedAgent.StartAsync(DatabaseProviders.Sqlite, types: ["page"]); + await agent.CreateAsPersonAsync("page"); + await agent.CreateAsPersonAsync("product"); + + var refused = await agent.FailAsync("search_entries", new() { ["query"] = "4411", ["types"] = new[] { "page", "prodcut", "product" } }); + + Assert.Equal( + [SchemaErrorCodes.ContentTypeNotFound, AgentPermissionErrorCodes.ScopeDenied], + refused["errors"]!.AsArray().Select(error => (string?)error!["code"])); + } + [Fact] public async Task A_query_without_words_an_unknown_state_or_an_empty_filter_is_a_tool_error() {