From cea7c7a87ddcd64dc2a6b81f863800556f707918 Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Mon, 19 Oct 2026 03:25:46 -0400 Subject: [PATCH 1/2] Filter search by field values and references, with text optional for one type --- CONTRIBUTING.md | 2 +- .../Builtin/Tools/SearchTools.cs | 83 +++++++- src/FluentCMS.Core/Search/FieldFilters.cs | 196 ++++++++++++++++++ src/FluentCMS.Core/Search/ISearch.cs | 26 +++ .../Tools/ToolArgumentsReader.cs | 16 ++ .../Search/EntrySearch.cs | 156 ++++++++++++-- .../Mcp/SearchToolsTests.cs | 46 +++- .../Search/SearchTests.cs | 72 +++++++ .../Search/FieldFiltersTests.cs | 114 ++++++++++ 9 files changed, 686 insertions(+), 25 deletions(-) create mode 100644 src/FluentCMS.Core/Search/FieldFilters.cs create mode 100644 tests/FluentCMS.Tests/Search/FieldFiltersTests.cs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c3d52d7c..1dc88449f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,7 +138,7 @@ An LLM-judged type asks a model only through `IModelGateway`, which it reads fro A check is a versioned definition under a slug: a check type and its settings, a description of what it enforces, a severity (block or warn), an optional scope of content types and fields, and test examples, at least one it must pass and one it must fail. `ICheckDirectory` activates a version only after running its examples, and names each example that does not get its verdict (CHK-02). A revision leaves the active version in place until the new one is activated, and only a person activates or deactivates a check. `ICheckRunner` runs the active checks that apply to a piece of content and gives each one's status: pass, warn for a failing check of warn severity, fail, or needs review. Agents run it on a draft with the `check_draft` tool before they propose it (CHK-04). -Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads the matches a page at a time, `Search__PageSize` (1,000 by default), and drops what the scope, deletions and filters leave out, until enough are left, the index has no more, or it has read `Search__MaxMatches` (10,000). A result that stopped there says it is not complete. Each hit has the state the entry tools show, in review included. A string that is a GUID, such as a reference, is not indexed. +Search (`ISearchIndex` in `FluentCMS.Core.Search`, SRC-01, ADR-0010) indexes each entry's latest version in the same unit of work as the write, as the lowercase words of every string and number in its fields (`SearchText`), so a result appears the moment its write commits. PostgreSQL keeps them in a `tsvector` with a GIN index and SQLite in an FTS5 table, both outside the EF model and created by their migrations; an external engine implements the same interface, registered with `AddFluentCmsExtension(ServiceLifetime.Scoped)` when it needs the request's scope. `ISearch` combines the matches with the entry rows, so deleting, restoring and publishing change results without touching the index, follows a match to the entries that reference it, and returns only types the caller may read. It reads the matches a page at a time, `Search__PageSize` (1,000 by default), and drops what the scope, deletions and filters leave out, until enough are left, the index has no more, or it has read `Search__MaxMatches` (10,000). A result that stopped there says it is not complete. Each hit has the state the entry tools show, in review included. A string that is a GUID, such as a reference, is not indexed. Field filters (`SearchQuery.Fields`) compare one type's latest field values, an entity's by its fact's value, and `SearchQuery.References` keeps entries whose latest version references an id; both are checked against the type before anything is read and applied to each page's candidates. With one type and either filter the text can be left out, and the candidates are then that type's entries. ## Code style diff --git a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs index d7fac739d..17bd1ef83 100644 --- a/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs +++ b/src/FluentCMS.Core/Builtin/Tools/SearchTools.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using System.Text.Json.Nodes; using FluentCMS.Core.Content; using FluentCMS.Core.Results; @@ -7,11 +8,33 @@ namespace FluentCMS.Core.Builtin.Tools; -// Finding entries and entities by their words (SRC-01, MCP-01), within the types the agent may read. +// Finding entries and entities by their words, field values and references (SRC-01, MCP-01), within the types the agent may read. public sealed class SearchTools : IToolProvider { + private const string FieldsForm = "[{\"field\": \"price\", \"operator\": \"less_than\", \"value\": 20}]"; + private static readonly string StateNames = string.Join(", ", Enum.GetValues().Select(state => ToolJson.Name(state))); + private static readonly string OperatorNames = string.Join(", ", Enum.GetValues().Select(FieldFilters.Name)); + + private static readonly JsonObject FieldsSchema = new() + { + ["type"] = "array", + ["minItems"] = 1, + ["items"] = new JsonObject + { + ["type"] = "object", + ["properties"] = new JsonObject + { + ["field"] = new JsonObject { ["type"] = "string", ["minLength"] = 1 }, + ["operator"] = new JsonObject { ["enum"] = new JsonArray([.. Enum.GetValues().Select(op => (JsonNode)FieldFilters.Name(op))]) }, + ["value"] = new JsonObject(), + }, + ["required"] = new JsonArray("field", "operator", "value"), + ["additionalProperties"] = false, + }, + }; + public IReadOnlyList Tools { get; } = [ new AgentTool( @@ -19,13 +42,24 @@ public sealed class SearchTools : IToolProvider "Search entries and entities", "Finds entries and entities whose latest version has every word of the query, each matched as a prefix, best first: " + "'4411' finds a SKU and '441' does too. An entry that references a match is found as well, with the match in 'via', " - + "such as the page that shows a matching entity. Narrow the search with types, states or published_only; read a hit " - + "with get_entry, or an entity's facts with get_entity_facts.", + + "such as the page that shows a matching entity. Narrow the search with types, states or published_only, with fields " + + "on one type's field values, such as price less_than 20, or with references to keep entries that reference an id. " + + "With one type and fields or references, the query can be left out. Read a hit with get_entry, or an entity's facts " + + "with get_entity_facts.", ToolSchemas.Object( - ToolSchemas.String("query", "The words to find, such as a name, a SKU or part of a sentence.", "4411"), + ToolSchemas.String("query", "The words to find, such as a name, a SKU or part of a sentence; optional with one type and fields or references.", "4411", required: false), ToolSchemas.Strings("types", "Content type slugs to search in; leave out for every type you may read.", "product", required: false), ToolSchemas.Strings("states", $"Entry states to keep: {StateNames}.", "published", required: false), ToolSchemas.Flag("published_only", "Only entries that have a published version."), + ToolSchemas.Json( + "fields", + $"Filters on the latest version's field values, all of which must hold; they need exactly one type in types. Operators: " + + $"{OperatorNames}. Text, rich text and enum fields take equal_to and contains, numbers and date-times the comparisons, " + + "booleans and references equal_to. An entity's field is compared by its fact's value.", + FieldsSchema, + JsonNode.Parse(FieldsForm)!, + required: false), + ToolSchemas.Uuid("references", "An entry or entity id; only entries whose latest version references it are kept.", required: false), ToolSchemas.PositiveInteger("limit", $"At most this many hits, up to {SearchQuery.MaxLimit}; {SearchQuery.DefaultLimit} if left out.", 20, required: false)), readOnly: true, SearchAsync), @@ -34,12 +68,15 @@ public sealed class SearchTools : IToolProvider private static async Task SearchAsync(ToolInvocation call, CancellationToken cancellationToken) { var arguments = new ToolArgumentsReader(call.Arguments); - var text = arguments.RequiredString("query", "4411"); + var text = arguments.OptionalString("query", "4411"); var types = arguments.OptionalStrings("types", "product"); var states = arguments.OptionalStrings("states", "published"); var publishedOnly = arguments.Flag("published_only"); var limit = arguments.OptionalPositiveInteger("limit") ?? SearchQuery.DefaultLimit; + var fields = arguments.OptionalArray("fields", FieldsForm); + var references = arguments.OptionalGuid("references"); var errors = new List(arguments.Errors()); + var filters = fields is { } items ? FieldFiltersOf(items, errors) : null; var parsedStates = new List(); foreach (var name in states ?? []) @@ -61,8 +98,10 @@ private static async Task SearchAsync(ToolInvocation call, Cancellat } var results = await call.Services.GetRequiredService().SearchAsync( - new SearchQuery(text!) + new SearchQuery(text ?? string.Empty) { + Fields = filters, + References = references, ContentTypes = types, States = states is null ? null : parsedStates, PublishedOnly = publishedOnly, @@ -89,4 +128,36 @@ private static async Task SearchAsync(ToolInvocation call, Cancellat : "Read a hit with get_entry, or an entity's facts with get_entity_facts. A hit with 'via' references that match.", }); } + + // Each item names a field, an operator by its tool name and a value. Whether the field exists and the value fits its type is + // checked against the type by the search. + private static List FieldFiltersOf(JsonElement items, List errors) + { + var filters = new List(); + var position = 0; + foreach (var item in items.EnumerateArray()) + { + var at = $"fields[{position++}]"; + if (item.ValueKind != JsonValueKind.Object + || item.EnumerateObject().Any(property => property.Name is not ("field" or "operator" or "value")) + || !item.TryGetProperty("field", out var field) || field.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(field.GetString()) + || !item.TryGetProperty("operator", out var name) || name.ValueKind != JsonValueKind.String + || !item.TryGetProperty("value", out var value) || value.ValueKind == JsonValueKind.Null) + { + errors.Add(new Error(ToolErrorCodes.ArgumentInvalid, $"{at} must be an object with a field, an operator and a value, and nothing else, such as {FieldsForm[1..^1]}.")); + continue; + } + + var operators = Enum.GetValues().Where(candidate => FieldFilters.Name(candidate) == name.GetString()).ToList(); + if (operators.Count != 1) + { + errors.Add(new Error(ToolErrorCodes.ArgumentInvalid, $"{at}: '{name.GetString()}' is not an operator; use {OperatorNames}.")); + continue; + } + + filters.Add(new FieldFilter(field.GetString()!, operators[0], value.Clone())); + } + + return filters; + } } diff --git a/src/FluentCMS.Core/Search/FieldFilters.cs b/src/FluentCMS.Core/Search/FieldFilters.cs new file mode 100644 index 000000000..bfc52eb76 --- /dev/null +++ b/src/FluentCMS.Core/Search/FieldFilters.cs @@ -0,0 +1,196 @@ +using System.Globalization; +using System.Text.Json; +using FluentCMS.Core.Results; +using FluentCMS.Core.Schema; + +namespace FluentCMS.Core.Search; + +// Field filters checked against one content type's definition, then applied to its entries' latest fields (SRC-01). Each field type +// takes the operators that mean something for it, and a value of its own kind; an entity's field is compared by its fact's value. +public sealed class FieldFilters +{ + private static readonly FieldOperator[] Ordering = + [FieldOperator.EqualTo, FieldOperator.LessThan, FieldOperator.AtMost, FieldOperator.GreaterThan, FieldOperator.AtLeast]; + + private readonly IReadOnlyList<(FieldFilter Filter, string Type)> _filters; + private readonly bool _facts; + + private FieldFilters(IReadOnlyList<(FieldFilter Filter, string Type)> filters, bool facts) + { + _filters = filters; + _facts = facts; + } + + // No field filters, so every entry's fields are accepted. + public static FieldFilters None { get; } = new([], facts: false); + + public static Result Create(IReadOnlyList filters, string contentType, ContentTypeDefinition definition) + { + ArgumentNullException.ThrowIfNull(filters); + ArgumentNullException.ThrowIfNull(definition); + + var errors = new List(); + var checkedFilters = new List<(FieldFilter, string)>(); + + foreach (var filter in filters) + { + var field = definition.Fields.FirstOrDefault(candidate => candidate.Name == filter.Field); + if (field is null) + { + errors.Add(new Error(SearchErrorCodes.FieldNotFound, $"'{contentType}' has no field '{filter.Field}'; get_content_type lists its fields.")); + continue; + } + + if (Operators(field.Type) is not { } operators) + { + errors.Add(new Error( + SearchErrorCodes.FieldNotFilterable, + $"The field '{field.Name}' is a {field.Type} field, which search does not filter on; filter on a text, rich text, enum, number, " + + "date-time, boolean or reference field.")); + continue; + } + + if (!operators.Contains(filter.Operator)) + { + errors.Add(new Error( + SearchErrorCodes.OperatorNotSupported, + $"'{Name(filter.Operator)}' does not apply to the {field.Type} field '{field.Name}'; use {string.Join(", ", operators.Select(Name))}.")); + continue; + } + + if (!ValueFits(field.Type, filter.Value)) + { + errors.Add(new Error( + SearchErrorCodes.FilterValueInvalid, + $"The value for the {field.Type} field '{field.Name}' must be {Expected(field.Type)}.")); + continue; + } + + checkedFilters.Add((filter, field.Type)); + } + + return errors.Count == 0 + ? Result.Success(new FieldFilters(checkedFilters, definition.IsEntity)) + : Result.Failure(errors); + } + + // Whether every filter holds for the fields; a field that is missing or null holds none. + public bool Matches(JsonElement fields) + { + foreach (var (filter, type) in _filters) + { + if (fields.ValueKind != JsonValueKind.Object || !fields.TryGetProperty(filter.Field, out var value)) + { + return false; + } + + if (_facts && value.ValueKind == JsonValueKind.Object && value.TryGetProperty("value", out var fact)) + { + value = fact; + } + + if (!Holds(type, filter, value)) + { + return false; + } + } + + return true; + } + + // The operator as tools and messages name it, such as less_than. + public static string Name(FieldOperator op) + { + return op switch + { + FieldOperator.EqualTo => "equal_to", + FieldOperator.Contains => "contains", + FieldOperator.LessThan => "less_than", + FieldOperator.AtMost => "at_most", + FieldOperator.GreaterThan => "greater_than", + _ => "at_least", + }; + } + + private static FieldOperator[]? Operators(string type) + { + return type switch + { + "text" or "richText" or "enum" => [FieldOperator.EqualTo, FieldOperator.Contains], + "number" or "dateTime" => Ordering, + "boolean" or "reference" => [FieldOperator.EqualTo], + _ => null, + }; + } + + private static bool ValueFits(string type, JsonElement value) + { + return type switch + { + "number" => value.ValueKind == JsonValueKind.Number, + "dateTime" => value.ValueKind == JsonValueKind.String && Moment(value.GetString()) is not null, + "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, + "reference" => value.ValueKind == JsonValueKind.String && Guid.TryParse(value.GetString(), out _), + _ => value.ValueKind == JsonValueKind.String && value.GetString()!.Length > 0, + }; + } + + private static string Expected(string type) + { + return type switch + { + "number" => "a number, such as 19.99", + "dateTime" => "an ISO 8601 date and time, such as \"2026-10-19T09:00:00Z\"", + "boolean" => "true or false", + "reference" => "the id of an entry or entity, such as \"0192f3a1-7c2e-7a4b-9d3e-1f2a3b4c5d6e\"", + _ => "a non-empty string", + }; + } + + private static bool Holds(string type, FieldFilter filter, JsonElement actual) + { + switch (type) + { + case "number": + return actual.ValueKind == JsonValueKind.Number && Compare(filter.Operator, actual.GetDouble().CompareTo(filter.Value.GetDouble())); + case "dateTime": + return actual.ValueKind == JsonValueKind.String + && Moment(actual.GetString()) is { } moment + && Compare(filter.Operator, moment.CompareTo(Moment(filter.Value.GetString())!.Value)); + case "boolean": + return actual.ValueKind == filter.Value.ValueKind; + case "reference": + return actual.ValueKind == JsonValueKind.String + && Guid.TryParse(actual.GetString(), out var id) + && id == Guid.Parse(filter.Value.GetString()!); + default: + if (actual.ValueKind != JsonValueKind.String) + { + return false; + } + + var text = actual.GetString()!; + var expected = filter.Value.GetString()!; + return filter.Operator == FieldOperator.Contains + ? text.Contains(expected, StringComparison.OrdinalIgnoreCase) + : string.Equals(text, expected, StringComparison.Ordinal); + } + } + + private static bool Compare(FieldOperator op, int comparison) + { + return op switch + { + FieldOperator.EqualTo => comparison == 0, + FieldOperator.LessThan => comparison < 0, + FieldOperator.AtMost => comparison <= 0, + FieldOperator.GreaterThan => comparison > 0, + _ => comparison >= 0, + }; + } + + private static DateTimeOffset? Moment(string? text) + { + return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var moment) ? moment : null; + } +} diff --git a/src/FluentCMS.Core/Search/ISearch.cs b/src/FluentCMS.Core/Search/ISearch.cs index e9b6b1a6d..95c3c38a8 100644 --- a/src/FluentCMS.Core/Search/ISearch.cs +++ b/src/FluentCMS.Core/Search/ISearch.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using FluentCMS.Core.Content; using FluentCMS.Core.Results; @@ -10,6 +11,7 @@ public interface ISearch Task> SearchAsync(SearchQuery query, CancellationToken cancellationToken = default); } +// Text is optional when the query names one content type and filters it by field values or a reference. public sealed record SearchQuery(string Text) { public const int DefaultLimit = 20; @@ -23,9 +25,28 @@ public sealed record SearchQuery(string Text) public bool PublishedOnly { get; init; } + // On the latest version's field values, all of which must hold; they need exactly one content type in ContentTypes. + public IReadOnlyList? Fields { get; init; } + + // An entry or entity id: only entries whose latest version references it. + public Guid? References { get; init; } + public int Limit { get; init; } = DefaultLimit; } +// On an entity, the field's fact value is compared. +public sealed record FieldFilter(string Field, FieldOperator Operator, JsonElement Value); + +public enum FieldOperator +{ + EqualTo, + Contains, + LessThan, + AtMost, + GreaterThan, + AtLeast, +} + // Complete is false when the search stopped at the most matches it reads, so better-ranked matches outside the filters or the // scope may have crowded out some that would pass them. public sealed record SearchResults(IReadOnlyList Hits, bool Complete = true); @@ -46,4 +67,9 @@ public static class SearchErrorCodes public const string QueryTooLong = "search_query_too_long"; public const string LimitInvalid = "search_limit_invalid"; public const string FilterEmpty = "search_filter_empty"; + public const string FilterNeedsOneType = "search_filter_needs_one_type"; + public const string FieldNotFound = "search_field_not_found"; + public const string FieldNotFilterable = "search_field_not_filterable"; + public const string OperatorNotSupported = "search_operator_not_supported"; + public const string FilterValueInvalid = "search_filter_value_invalid"; } diff --git a/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs b/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs index fdc948a7c..019eabea9 100644 --- a/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs +++ b/src/FluentCMS.Core/Tools/ToolArgumentsReader.cs @@ -94,6 +94,22 @@ public ToolArgumentsReader(JsonElement arguments) return null; } + public JsonElement? OptionalArray(string name, string form) + { + if (Read(name) is not { } value) + { + return null; + } + + if (value.ValueKind == JsonValueKind.Array) + { + return value; + } + + Invalid(name, $"must be a JSON array: {form}"); + return null; + } + public bool Flag(string name) { if (Read(name) is not { } value) diff --git a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs index b206b0e78..e7d0cef0c 100644 --- a/src/FluentCMS.Infrastructure/Search/EntrySearch.cs +++ b/src/FluentCMS.Infrastructure/Search/EntrySearch.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using FluentCMS.Core.Agents; using FluentCMS.Core.Content; using FluentCMS.Core.Results; @@ -13,7 +14,13 @@ namespace FluentCMS.Infrastructure.Search; // The index finds the text; the entry rows decide the rest, so deleting, restoring and publishing change results without touching // the index. A match is followed to the entries whose latest version references it, such as the page that shows a matching entity. -internal sealed class EntrySearch(FluentCmsDbContext context, ISearchIndex index, IAgentPermissions permissions, IOptions options) : ISearch +// Field and reference filters read the latest versions of the candidates that pass everything else. +internal sealed class EntrySearch( + FluentCmsDbContext context, + ISearchIndex index, + IAgentPermissions permissions, + IContentTypeDirectory contentTypes, + IOptions options) : ISearch { private readonly SearchOptions _settings = options.Value; @@ -35,17 +42,24 @@ public async Task> SearchAsync(SearchQuery query, Cancella return Result.Failure(types); } - var (matched, ranks, complete) = await MatchedAsync(terms, readable, query, cancellationToken); + var filters = await FieldFiltersAsync(query, cancellationToken); + if (!filters.IsSuccess) + { + return Result.Failure(filters.Errors); + } + + var (matched, passing, ranks, complete) = await MatchedAsync(terms, readable, query, filters.Value, cancellationToken); var direct = matched - .Where(entry => Passes(entry, query)) + .Where(entry => passing.Contains(entry.Id)) .Select(entry => entry.Hit(ranks[entry.Id], via: null)) .OrderByDescending(hit => hit.Rank) .ThenBy(hit => hit.EntryId) .Take(query.Limit) .ToList(); + // Without words every candidate is a match, so there is nothing for a reference to explain. var room = query.Limit - direct.Count; - if (room == 0) + if (room == 0 || terms.Count == 0) { return Result.Success(new SearchResults(direct, complete)); } @@ -63,8 +77,12 @@ public async Task> SearchAsync(SearchQuery query, Cancella var referencingIds = references.Select(reference => reference.EntryId).Distinct().ToList(); var shown = direct.Select(hit => hit.EntryId).ToHashSet(); - var via = (await EntriesAsync(context.Entries.Where(row => referencingIds.Contains(row.Id)), cancellationToken)) + var referencing = (await EntriesAsync(context.Entries.Where(row => referencingIds.Contains(row.Id)), cancellationToken)) .Where(entry => !shown.Contains(entry.Id) && readable(entry.ContentType) && Passes(entry, query)) + .ToList(); + var referencingPassing = await StructuredAsync(referencing, query, filters.Value, cancellationToken); + var via = referencing + .Where(entry => referencingPassing.Contains(entry.Id)) .Select(entry => { // The best match the entry references explains why it is shown. @@ -85,35 +103,125 @@ public async Task> SearchAsync(SearchQuery query, Cancella // The readable matches, a page at a time in rank order. Better-ranked matches the scope, deletions or filters drop would otherwise // crowd out the rest, so reading stops only once enough pass the filters, the index has no more, or MaxMatches are read. - private async Task<(List Matched, Dictionary Ranks, bool Complete)> MatchedAsync( + private async Task<(List Matched, HashSet Passing, Dictionary Ranks, bool Complete)> MatchedAsync( IReadOnlyList terms, Func readable, SearchQuery query, + FieldFilters filters, CancellationToken cancellationToken) { var matched = new List(); + var passing = new HashSet(); var ranks = new Dictionary(); var read = 0; while (read < _settings.MaxMatches) { var take = Math.Min(_settings.PageSize, _settings.MaxMatches - read); - var page = await index.MatchAsync(terms, read, take, cancellationToken); + var page = await PageAsync(terms, query, read, take, cancellationToken); read += page.Count; var fresh = page.Where(match => ranks.TryAdd(match.EntryId, match.Rank)).Select(match => match.EntryId).ToList(); - matched.AddRange((await EntriesAsync(context.Entries.Where(row => fresh.Contains(row.Id)), cancellationToken)) - .Where(entry => readable(entry.ContentType))); + var candidates = (await EntriesAsync(context.Entries.Where(row => fresh.Contains(row.Id)), cancellationToken)) + .Where(entry => readable(entry.ContentType)) + .ToList(); + matched.AddRange(candidates); + passing.UnionWith(await StructuredAsync([.. candidates.Where(entry => Passes(entry, query))], query, filters, cancellationToken)); - if (page.Count < take || matched.Count(entry => Passes(entry, query)) >= query.Limit) + if (page.Count < take || passing.Count >= query.Limit) { - return (matched, ranks, true); + return (matched, passing, ranks, true); } } // Whether anything is left past the last match read. - var more = await index.MatchAsync(terms, read, 1, cancellationToken); - return (matched, ranks, more.Count == 0); + var more = await PageAsync(terms, query, read, 1, cancellationToken); + return (matched, passing, ranks, more.Count == 0); + } + + // Matches of the words in rank order, or without words the one type's entries in id order, each with no rank. + private async Task> PageAsync(IReadOnlyList terms, SearchQuery query, int skip, int take, CancellationToken cancellationToken) + { + if (terms.Count > 0) + { + return await index.MatchAsync(terms, skip, take, cancellationToken); + } + + var siteId = Site.Default.Id; + var slug = query.ContentTypes![0]; + var ids = await context.Entries + .AsNoTracking() + .Join(context.ContentTypes, row => row.ContentTypeId, type => type.Id, (row, type) => new { row, type.Slug }) + .Where(pair => pair.row.SiteId == siteId && pair.Slug == slug && pair.row.DeletedAt == null) + .OrderBy(pair => pair.row.Id) + .Skip(skip) + .Take(take) + .Select(pair => pair.row.Id) + .ToListAsync(cancellationToken); + + return [.. ids.Select(id => new SearchMatch(id, 0))]; + } + + // The field filters checked against the one type they need, before anything is read. + private async Task> FieldFiltersAsync(SearchQuery query, CancellationToken cancellationToken) + { + if (query.Fields is not { Count: > 0 } fields) + { + return Result.Success(FieldFilters.None); + } + + var slug = query.ContentTypes![0]; + var type = await contentTypes.GetAsync(slug, cancellationToken); + + return type.IsSuccess ? FieldFilters.Create(fields, slug, type.Value.Definition) : Result.Failure(type.Errors); + } + + // The candidates whose latest version references the given id and has field values the filters accept; every candidate when the + // query has neither. + private async Task> StructuredAsync(IReadOnlyList candidates, SearchQuery query, FieldFilters filters, CancellationToken cancellationToken) + { + var ids = candidates.Select(entry => entry.Id).ToList(); + if (ids.Count == 0 || !IsStructured(query)) + { + return [.. ids]; + } + + if (query.References is { } target) + { + ids = await context.EntryReferences + .AsNoTracking() + .Where(reference => ids.Contains(reference.EntryId) && reference.TargetId == target) + .Join(context.Entries, reference => reference.EntryId, row => row.Id, (reference, row) => new { reference, row }) + .Where(pair => pair.reference.Version == pair.row.LatestVersion) + .Select(pair => pair.reference.EntryId) + .Distinct() + .ToListAsync(cancellationToken); + } + + if (query.Fields is { Count: > 0 } && ids.Count > 0) + { + var versions = await context.EntryVersions + .AsNoTracking() + .Join(context.Entries, version => version.EntryId, row => row.Id, (version, row) => new { version, row }) + .Where(pair => ids.Contains(pair.row.Id) && pair.version.Version == pair.row.LatestVersion) + .Select(pair => new { pair.row.Id, pair.version.Fields }) + .ToListAsync(cancellationToken); + + ids = [.. versions.Where(version => Accepts(filters, version.Fields)).Select(version => version.Id)]; + } + + return [.. ids]; + } + + private static bool Accepts(FieldFilters filters, string fields) + { + using var document = JsonDocument.Parse(fields); + return filters.Matches(document.RootElement); + } + + private static bool IsStructured(SearchQuery query) + { + return query.References is not null || query.Fields is { Count: > 0 }; } // Each type a filter names exists and is one the caller may read, as list_entries asks, so a typo or a type outside the scope is @@ -187,9 +295,11 @@ private static List Validate(SearchQuery query, IReadOnlyList ter { var errors = new List(); - if (terms.Count == 0) + if (terms.Count == 0 && !IsStructured(query)) { - errors.Add(new Error(SearchErrorCodes.QueryEmpty, "The query has no words to search for; give at least one word or number, such as '4411'.")); + errors.Add(new Error( + SearchErrorCodes.QueryEmpty, + "The query has no words to search for; give at least one word or number, such as '4411', or filter one type by its fields or a reference.")); } else if (terms.Count > SearchTerms.MaxTerms || terms.Any(term => term.Length > SearchTerms.MaxTermLength)) { @@ -203,6 +313,22 @@ private static List Validate(SearchQuery query, IReadOnlyList ter errors.Add(new Error(SearchErrorCodes.FilterEmpty, "The types filter is empty; name at least one content type, or leave it out to search every type you may read.")); } + if (query.Fields is { Count: 0 }) + { + errors.Add(new Error(SearchErrorCodes.FilterEmpty, "The fields filter is empty; give at least one field filter, or leave it out.")); + } + + // A field belongs to one type, and without words the candidates are one type's entries. + var needsOneType = query.Fields is { Count: > 0 } || (terms.Count == 0 && IsStructured(query)); + if (needsOneType && query.ContentTypes is not { Count: 0 } && query.ContentTypes?.Distinct(StringComparer.Ordinal).Count() != 1) + { + errors.Add(new Error( + SearchErrorCodes.FilterNeedsOneType, + terms.Count == 0 + ? "A search without words needs exactly one type in types, such as [\"product\"], whose entries it filters." + : "Field filters need exactly one type in types, such as [\"product\"], whose fields they name.")); + } + if (query.States is { Count: 0 }) { errors.Add(new Error(SearchErrorCodes.FilterEmpty, "The states filter is empty; name at least one state, or leave it out to keep every state.")); diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs index acd8d5f0c..429c440a3 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -82,6 +82,41 @@ await agent.AsPersonAsync(directory => Assert.Equal([(product, (Guid?)null), (page, product)], person.Hits.Select(hit => (hit.EntryId, hit.Via))); } + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task An_agent_filters_by_field_values_and_references_with_or_without_words(string provider) + { + await using var agent = await ConnectedAgent.StartAsync(provider); + await agent.AsPersonAsync(directory => + directory.CreateAsync(new NewContentType("product", "product", new ContentTypeDefinition( + [new FieldDefinition("title", "text"), new FieldDefinition("price", "number")])))); + await agent.AsPersonAsync(directory => + directory.CreateAsync(new NewContentType("page", "page", new ContentTypeDefinition( + [new FieldDefinition("title", "text"), new FieldDefinition("product", "reference", settings: JsonSerializer.SerializeToElement(new { target = "product" }))])))); + var cheap = await CreateAsync(agent, "product", new { title = "Part 4411", price = 19.99 }); + await CreateAsync(agent, "product", new { title = "Part 4412", price = 25 }); + var page = await CreateAsync(agent, "page", new { title = "Spring sale", product = cheap.ToString() }); + + var underTwenty = await agent.CallAsync("search_entries", new() + { + ["query"] = "part", + ["types"] = new[] { "product" }, + ["fields"] = new[] { new { field = "price", @operator = "less_than", value = 20 } }, + }); + var referencing = await agent.CallAsync("search_entries", new() { ["types"] = new[] { "page" }, ["references"] = cheap.ToString() }); + var refused = await agent.FailAsync("search_entries", new() + { + ["types"] = new[] { "product" }, + ["fields"] = new object[] { new { field = "price", @operator = "below", value = 20 }, new { field = "price" } }, + }); + + Assert.Equal([cheap.ToString()], underTwenty["hits"]!.AsArray().Select(hit => (string?)hit!["id"])); + Assert.Equal([page.ToString()], referencing["hits"]!.AsArray().Select(hit => (string?)hit!["id"])); + Assert.Equal([ToolErrorCodes.ArgumentInvalid, ToolErrorCodes.ArgumentInvalid], refused["errors"]!.AsArray().Select(error => (string?)error!["code"])); + Assert.Contains("'below' is not an operator", (string?)refused["errors"]![0]!["message"]); + } + [Fact] public async Task A_type_filter_naming_an_unknown_type_or_one_outside_the_scope_is_a_tool_error() { @@ -114,9 +149,14 @@ public async Task A_query_without_words_an_unknown_state_or_an_empty_filter_is_a Assert.Contains("'live' is not an entry state", (string?)state["errors"]![0]!["message"]); } - private static async Task CreateAsync(ConnectedAgent agent, string type, string title) + private static Task CreateAsync(ConnectedAgent agent, string type, string title) + { + return CreateAsync(agent, type, new { title }); + } + + private static async Task CreateAsync(ConnectedAgent agent, string type, object fields) { - var fields = JsonSerializer.SerializeToElement(new { title }); - return (await agent.AsPersonAsync(entries => entries.CreateAsync(new NewEntry(type, fields)))).Id; + var json = JsonSerializer.SerializeToElement(fields); + return (await agent.AsPersonAsync(entries => entries.CreateAsync(new NewEntry(type, json)))).Id; } } diff --git a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs index c21a0c881..05964a56e 100644 --- a/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs +++ b/tests/FluentCMS.IntegrationTests/Search/SearchTests.cs @@ -161,6 +161,78 @@ public async Task A_filtered_search_reads_past_better_ranked_matches_and_says_wh Assert.Equal((5, true), (unfiltered.Value.Hits.Count, unfiltered.Value.Complete)); } + // An entity's price is a fact, so the filter compares the fact's value; text and filters both have to hold. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task Field_filters_narrow_a_text_search_and_filter_one_type_without_words(string provider) + { + await using var scenario = await EntryScenario.CreateAsync(provider); + await scenario.Schema.CreateEntityAsync("product", Field("name", "text"), Field("price", "number")); + var cheap = (await scenario.CreateAsync("product", $$"""{"name": {{Fact("\"Widget\"")}}, "price": {{Fact("19.99")}}}""")).Value.Id; + await scenario.CreateAsync("product", $$"""{"name": {{Fact("\"Widget Pro\"")}}, "price": {{Fact("25")}}}"""); + await scenario.CreateAsync("product", $$"""{"name": {{Fact("\"Gadget\"")}}, "price": {{Fact("5")}}}"""); + + var underTwenty = await SearchAsync(scenario, new SearchQuery("widget") + { + ContentTypes = ["product"], + Fields = [new FieldFilter("price", FieldOperator.LessThan, Json("20"))], + }); + var noWords = await SearchAsync(scenario, new SearchQuery(string.Empty) + { + ContentTypes = ["product"], + Fields = [new FieldFilter("name", FieldOperator.Contains, Json("\"wid\""))], + }); + + Assert.Equal([cheap], underTwenty.Select(hit => hit.EntryId)); + Assert.Equal(2, noWords.Count); + Assert.All(noWords, hit => Assert.Equal(0, hit.Rank)); + } + + // Only the latest version counts, so a page that referenced the product before an update is left out. + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task A_reference_filter_keeps_the_entries_whose_latest_version_references_the_id(string provider) + { + await using var scenario = await CatalogAsync(provider); + var earlier = (await scenario.Entries.CreateAsync("page", $$"""{"title": "Last season", "product": "{{scenario.Product}}"}""")).Value.Id; + await scenario.Entries.UpdateAsync(earlier, 1, """{"title": "Last season", "product": null}"""); + + var referencing = await SearchAsync(scenario.Entries, new SearchQuery(string.Empty) { ContentTypes = ["page"], References = scenario.Product }); + var withWords = await SearchAsync(scenario.Entries, new SearchQuery("seller") { References = scenario.Product }); + var otherWords = await SearchAsync(scenario.Entries, new SearchQuery("hours") { References = scenario.Product }); + + Assert.Equal([scenario.Page], referencing.Select(hit => hit.EntryId)); + Assert.Equal([scenario.Page], withWords.Select(hit => hit.EntryId)); + Assert.Empty(otherWords); + } + + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task Field_filters_that_name_no_single_type_or_do_not_fit_their_field_are_refused(string provider) + { + await using var scenario = await CatalogAsync(provider); + var byName = new FieldFilter("name", FieldOperator.Contains, Json("\"wid\"")); + + var noType = await RunAsync(scenario.Entries, new SearchQuery("widget") { Fields = [byName] }); + var twoTypes = await RunAsync(scenario.Entries, new SearchQuery("widget") { ContentTypes = ["product", "page"], Fields = [byName] }); + var noWordsNoType = await RunAsync(scenario.Entries, new SearchQuery(string.Empty) { References = scenario.Product }); + var emptyFields = await RunAsync(scenario.Entries, new SearchQuery("widget") { ContentTypes = ["product"], Fields = [] }); + var unknownField = await RunAsync(scenario.Entries, new SearchQuery("widget") + { + ContentTypes = ["product"], + Fields = [new FieldFilter("colour", FieldOperator.EqualTo, Json("\"red\""))], + }); + + Assert.Equal(SearchErrorCodes.FilterNeedsOneType, Assert.Single(noType.Errors).Code); + Assert.Equal(SearchErrorCodes.FilterNeedsOneType, Assert.Single(twoTypes.Errors).Code); + Assert.Equal(SearchErrorCodes.FilterNeedsOneType, Assert.Single(noWordsNoType.Errors).Code); + Assert.Equal(SearchErrorCodes.FilterEmpty, Assert.Single(emptyFields.Errors).Code); + Assert.Equal(SearchErrorCodes.FieldNotFound, Assert.Single(unknownField.Errors).Code); + } + [Theory] [InlineData(DatabaseProviders.Sqlite)] [PostgresData] diff --git a/tests/FluentCMS.Tests/Search/FieldFiltersTests.cs b/tests/FluentCMS.Tests/Search/FieldFiltersTests.cs new file mode 100644 index 000000000..7095b2145 --- /dev/null +++ b/tests/FluentCMS.Tests/Search/FieldFiltersTests.cs @@ -0,0 +1,114 @@ +using System.Text.Json; +using FluentCMS.Core.Schema; +using FluentCMS.Core.Search; + +namespace FluentCMS.Tests.Search; + +public sealed class FieldFiltersTests +{ + private static readonly ContentTypeDefinition Product = new( + [ + new FieldDefinition("name", "text"), + new FieldDefinition("price", "number"), + new FieldDefinition("launch", "dateTime"), + new FieldDefinition("active", "boolean"), + new FieldDefinition("category", "reference", settings: Json("""{"target": "category"}""")), + new FieldDefinition("tags", "list", settings: Json("""{"itemType": "text"}""")), + ]); + + [Fact] + public void Each_field_type_takes_its_own_operators_and_a_value_of_its_kind() + { + var refused = FieldFilters.Create( + [ + Filter("colour", FieldOperator.EqualTo, "\"red\""), + Filter("tags", FieldOperator.EqualTo, "\"sale\""), + Filter("price", FieldOperator.Contains, "\"19\""), + Filter("price", FieldOperator.LessThan, "\"20\""), + Filter("launch", FieldOperator.AtLeast, "\"next spring\""), + Filter("active", FieldOperator.EqualTo, "\"yes\""), + Filter("category", FieldOperator.EqualTo, "\"not-an-id\""), + Filter("name", FieldOperator.Contains, "\"\""), + ], + "product", + Product); + + Assert.Equal( + [ + SearchErrorCodes.FieldNotFound, + SearchErrorCodes.FieldNotFilterable, + SearchErrorCodes.OperatorNotSupported, + SearchErrorCodes.FilterValueInvalid, + SearchErrorCodes.FilterValueInvalid, + SearchErrorCodes.FilterValueInvalid, + SearchErrorCodes.FilterValueInvalid, + SearchErrorCodes.FilterValueInvalid, + ], + refused.Errors.Select(error => error.Code)); + Assert.Contains("use equal_to, less_than, at_most, greater_than, at_least", refused.Errors[2].Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("name", FieldOperator.EqualTo, "\"Widget Pro\"", true)] + [InlineData("name", FieldOperator.EqualTo, "\"widget pro\"", false)] + [InlineData("name", FieldOperator.Contains, "\"WIDG\"", true)] + [InlineData("price", FieldOperator.LessThan, "20", true)] + [InlineData("price", FieldOperator.AtMost, "19.99", true)] + [InlineData("price", FieldOperator.GreaterThan, "19.99", false)] + [InlineData("price", FieldOperator.AtLeast, "19.99", true)] + [InlineData("launch", FieldOperator.LessThan, "\"2026-11-01T00:00:00Z\"", true)] + [InlineData("launch", FieldOperator.GreaterThan, "\"2026-11-01T00:00:00+02:00\"", false)] + [InlineData("active", FieldOperator.EqualTo, "true", true)] + [InlineData("active", FieldOperator.EqualTo, "false", false)] + [InlineData("category", FieldOperator.EqualTo, "\"0192F3A1-7C2E-7A4B-9D3E-1F2A3B4C5D6E\"", true)] + public void A_filter_holds_by_its_operator(string field, FieldOperator op, string value, bool holds) + { + var filters = FieldFilters.Create([Filter(field, op, value)], "product", Product).Value; + + Assert.Equal(holds, filters.Matches(Json(""" + { + "name": "Widget Pro", + "price": 19.99, + "launch": "2026-10-19T09:00:00Z", + "active": true, + "category": "0192f3a1-7c2e-7a4b-9d3e-1f2a3b4c5d6e", + "tags": ["sale"] + } + """))); + } + + [Fact] + public void A_missing_or_null_field_holds_no_filter_and_every_filter_must_hold() + { + var filters = FieldFilters.Create( + [Filter("price", FieldOperator.LessThan, "20"), Filter("name", FieldOperator.Contains, "\"pro\"")], + "product", + Product).Value; + + Assert.True(filters.Matches(Json("""{"name": "Widget Pro", "price": 10}"""))); + Assert.False(filters.Matches(Json("""{"name": "Widget", "price": 10}"""))); + Assert.False(filters.Matches(Json("""{"name": "Widget Pro", "price": null}"""))); + Assert.False(filters.Matches(Json("""{"name": "Widget Pro"}"""))); + } + + [Fact] + public void An_entitys_field_is_compared_by_its_facts_value() + { + var entity = new ContentTypeDefinition([new FieldDefinition("price", "number")], isEntity: true); + var filters = FieldFilters.Create([Filter("price", FieldOperator.LessThan, "20")], "product", entity).Value; + + Assert.True(filters.Matches(Json("""{"price": {"value": 19.99, "source": null, "owner": null}}"""))); + Assert.False(filters.Matches(Json("""{"price": {"value": 25, "source": null, "owner": null}}"""))); + Assert.True(FieldFilters.None.Matches(Json("{}"))); + } + + private static FieldFilter Filter(string field, FieldOperator op, string value) + { + return new FieldFilter(field, op, Json(value)); + } + + private static JsonElement Json(string json) + { + return JsonSerializer.Deserialize(json); + } +} From 52d7ba2352133aeccb1fa8f8630178d7ebfe1b39 Mon Sep 17 00:00:00 2001 From: Amir Pournasserian Date: Mon, 19 Oct 2026 04:41:02 -0400 Subject: [PATCH 2/2] Run the tool's filter refusals on both providers --- tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs index 97bda0961..0dfae5fbb 100644 --- a/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs +++ b/tests/FluentCMS.IntegrationTests/Mcp/SearchToolsTests.cs @@ -128,10 +128,12 @@ await agent.AsPersonAsync(entries => Assert.Contains("'below' is not an operator", (string?)refused["errors"]![0]!["message"]); } - [Fact] - public async Task Field_filters_that_do_not_fit_one_type_are_tool_errors_with_their_codes() + [Theory] + [InlineData(DatabaseProviders.Sqlite)] + [PostgresData] + public async Task Field_filters_that_do_not_fit_one_type_are_tool_errors_with_their_codes(string provider) { - await using var agent = await ConnectedAgent.StartAsync(DatabaseProviders.Sqlite); + await using var agent = await ConnectedAgent.StartAsync(provider); await agent.AsPersonAsync(directory => directory.CreateAsync(new NewContentType("product", "product", new ContentTypeDefinition( [new FieldDefinition("title", "text"), new FieldDefinition("price", "number")]))));