From 5330be8f724916471e1c2c0c4cc7f6c448761d56 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 22 Jul 2026 16:56:11 +0800 Subject: [PATCH 01/15] Add EventLogItems entity and related configurations, migrations for SQL Server, and tests --- .../20260722081605_EventLogItems.Designer.cs | 130 +++++++++++++ .../20260722081605_EventLogItems.cs | 47 +++++ ...verServiceControlDbContextModelSnapshot.cs | 47 +++++ .../DbContexts/ServiceControlDbContext.cs | 2 + .../Entities/EventLogItemEntity.cs | 24 +++ .../EventLogItemConfiguration.cs | 28 +++ .../EventLogDataStoreTests.cs | 182 ++++++++++++++++++ 7 files changed, 460 insertions(+) create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs create mode 100644 src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs new file mode 100644 index 0000000000..f7495d6209 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs @@ -0,0 +1,130 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260722081605_EventLogItems")] + partial class EventLogItems + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("nvarchar(600)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("KnownEndpointId") + .HasColumnType("uniqueidentifier"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.ToTable("KnownEndpointsInsertOnly", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs new file mode 100644 index 0000000000..7c3dbacbc8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs @@ -0,0 +1,47 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class EventLogItems : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "EventLogItems", + columns: table => new + { + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + EventLogItemId = table.Column(type: "nvarchar(600)", maxLength: 600, nullable: false), + Description = table.Column(type: "nvarchar(max)", nullable: false), + Severity = table.Column(type: "nvarchar(20)", maxLength: 20, nullable: false), + RaisedAt = table.Column(type: "datetime2", nullable: false), + RelatedTo = table.Column(type: "nvarchar(max)", nullable: false), + Category = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), + EventType = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_EventLogItems", x => x.Id); + }); + + migrationBuilder.CreateIndex( + name: "IX_EventLogItems_RaisedAt_Id", + table: "EventLogItems", + columns: new[] { "RaisedAt", "Id" }, + descending: new bool[0]); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "EventLogItems"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index 2539cbd49d..7fc280a0d8 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -227,6 +227,53 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("FailedMessageRetries"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("nvarchar(600)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index fa10d2e455..b6153edd23 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -14,6 +14,7 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont public DbSet FailedErrorImports { get; set; } public DbSet TrialMetadata { get; set; } public DbSet Subscriptions { get; set; } + public DbSet EventLogItems { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); @@ -30,6 +31,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new KnownEndpointConfiguration()); modelBuilder.ApplyConfiguration(new SubscriptionConfiguration()); modelBuilder.ApplyConfiguration(new TrialMetadataConfiguration()); + modelBuilder.ApplyConfiguration(new EventLogItemConfiguration()); } public abstract bool IsDuplicateKeyException(DbUpdateException exception); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs new file mode 100644 index 0000000000..d936c44f0d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +using ServiceControl.EventLog; + +public class EventLogItemEntity +{ + public long Id { get; set; } + + // The API-visible identity, assigned by EventLogMappingDefinition as + // "EventLogItem/{Category}/{EventType}/{guid}" + public required string EventLogItemId { get; set; } + + public required string Description { get; set; } + + public Severity Severity { get; set; } + + public DateTime RaisedAt { get; set; } + + public List RelatedTo { get; set; } = []; + + public required string Category { get; set; } + + public required string EventType { get; set; } +} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs new file mode 100644 index 0000000000..a6a39cbf01 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class EventLogItemConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("EventLogItems"); + builder.HasKey(e => e.Id); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + // Long enough for "EventLogItem/{Category}/{EventType}/{guid}" form. Deliberately not + // indexed and not unique. Nothing looks an item up by it, and each id embeds a fresh Guid. + builder.Property(e => e.EventLogItemId).IsRequired().HasMaxLength(600); + builder.Property(e => e.Description).IsRequired(); + builder.Property(e => e.Severity).IsRequired().HasConversion().HasMaxLength(20); + builder.Property(e => e.RaisedAt).IsRequired(); + // RelatedTo is only ever returned to the API, never queried, so it does not warrant a child table. + builder.Property(e => e.RelatedTo).IsRequired(); + builder.Property(e => e.Category).IsRequired().HasMaxLength(255); + builder.Property(e => e.EventType).IsRequired().HasMaxLength(255); + // Every read is "order by RaisedAt descending" plus paging. The key is included as a + // tiebreaker so that items sharing a RaisedAt do not shuffle between pages. + builder.HasIndex(e => new { e.RaisedAt, e.Id }).IsDescending(); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs new file mode 100644 index 0000000000..85488a541b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs @@ -0,0 +1,182 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.EventLog; +using ServiceControl.Persistence.Infrastructure; + +class EventLogDataStoreTests : PersistenceTestBase +{ + [Test] + public async Task Added_item_is_returned_with_all_values_intact() + { + var raisedAt = new DateTime(2026, 7, 22, 10, 30, 0, DateTimeKind.Utc); + var logItem = CreateLogItem("MessageFailed", raisedAt); + logItem.Severity = Severity.Error; + logItem.Description = "Message processing failed"; + logItem.RelatedTo = ["/message/abc123", "/endpoint/Sales"]; + + await EventLogDataStore.Add(logItem); + await CompleteDatabaseOperation(); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(items, Has.Count.EqualTo(1)); + var stored = items[0]; + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Id, Is.EqualTo(logItem.Id)); + Assert.That(stored.Description, Is.EqualTo("Message processing failed")); + Assert.That(stored.Severity, Is.EqualTo(Severity.Error)); + Assert.That(stored.RaisedAt, Is.EqualTo(raisedAt)); + Assert.That(stored.Category, Is.EqualTo(logItem.Category)); + Assert.That(stored.EventType, Is.EqualTo("MessageFailed")); + Assert.That(stored.RelatedTo, Is.EqualTo(["/message/abc123", "/endpoint/Sales"])); + } + } + + [Test] + public async Task Item_with_no_related_links_round_trips_as_empty() + { + var logItem = CreateLogItem("EndpointStarted", DateTime.UtcNow); + logItem.RelatedTo = []; + + await EventLogDataStore.Add(logItem); + await CompleteDatabaseOperation(); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(items[0].RelatedTo, Is.Empty); + } + + [Test] + public async Task Items_are_returned_most_recently_raised_first() + { + var baseTime = new DateTime(2026, 7, 22, 9, 0, 0, DateTimeKind.Utc); + await EventLogDataStore.Add(CreateLogItem("Oldest", baseTime)); + await EventLogDataStore.Add(CreateLogItem("Newest", baseTime.AddMinutes(2))); + await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1))); + await CompleteDatabaseOperation(); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(items.Select(i => i.EventType), Is.EqualTo(new[] { "Newest", "Middle", "Oldest" })); + } + + [Test] + public async Task Empty_store_returns_no_items() + { + var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(items, Is.Empty); + Assert.That(total, Is.Zero); + } + } + + [Test] + public async Task Page_size_limits_returned_items_but_not_the_total() + { + await AddItems(5); + + var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(items, Has.Count.EqualTo(2)); + Assert.That(total, Is.EqualTo(5)); + } + } + + [Test] + public async Task Later_pages_continue_where_the_previous_page_ended() + { + await AddItems(5); + + var (firstPage, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2)); + var (secondPage, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 2, pageSize: 2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(secondPage, Has.Count.EqualTo(2)); + Assert.That(secondPage.Select(i => i.Id).Intersect(firstPage.Select(i => i.Id)), Is.Empty); + } + } + + [Test] + public async Task Final_page_returns_only_the_remaining_items() + { + await AddItems(5); + + var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 3, pageSize: 2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(items, Has.Count.EqualTo(1)); + Assert.That(total, Is.EqualTo(5)); + } + } + + [Test] + public async Task Page_beyond_the_end_returns_no_items() + { + await AddItems(2); + + var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 5, pageSize: 2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(items, Is.Empty); + Assert.That(total, Is.EqualTo(2)); + } + } + + [Test] + public async Task Version_changes_when_an_item_is_added() + { + await AddItems(1); + var (_, _, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + await AddItems(1); + var (_, _, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(versionAfter, Is.Not.EqualTo(versionBefore)); + } + + [Test] + public async Task Version_is_stable_while_nothing_changes() + { + await AddItems(2); + + var (_, _, firstRead) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var (_, _, secondRead) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(secondRead, Is.EqualTo(firstRead)); + } + + async Task AddItems(int count) + { + var baseTime = new DateTime(2026, 7, 22, 8, 0, 0, DateTimeKind.Utc); + + for (var i = 0; i < count; i++) + { + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); + } + + await CompleteDatabaseOperation(); + } + + static EventLogItem CreateLogItem(string eventType, DateTime raisedAt) => new() + { + Id = $"EventLogItem/Recoverability/{eventType}/{Guid.NewGuid()}", + Category = "Recoverability", + EventType = eventType, + Description = $"{eventType} occurred", + Severity = Severity.Info, + RaisedAt = raisedAt, + RelatedTo = [] + }; +} From a730c18d3a263a131b0fbe0cd9fcb15b9eb93dae Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 27 Jul 2026 13:59:56 +0800 Subject: [PATCH 02/15] Update sql migrations. Add postgres migrations --- .../20260727055524_EventLogItems.Designer.cs | 322 ++++++++++++++++++ .../20260727055524_EventLogItems.cs | 49 +++ ...SqlServiceControlDbContextModelSnapshot.cs | 58 ++++ .../20260722081605_EventLogItems.Designer.cs | 130 ------- .../20260727055119_EventLogItems.Designer.cs | 261 ++++++++++++++ ...ems.cs => 20260727055119_EventLogItems.cs} | 2 +- ...verServiceControlDbContextModelSnapshot.cs | 94 ++--- .../EventLogItemConfiguration.cs | 5 +- 8 files changed, 741 insertions(+), 180 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260722081605_EventLogItems.cs => 20260727055119_EventLogItems.cs} (97%) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs new file mode 100644 index 0000000000..4c8c9d4342 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs @@ -0,0 +1,322 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260727055524_EventLogItems")] + partial class EventLogItems + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("character varying(600)") + .HasColumnName("event_log_item_id"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("EventLogItems", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("queue_address"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("QueueAddress") + .HasDatabaseName("ix_failed_messages_queue_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastModified") + .HasDatabaseName("ix_failed_messages_status_last_modified"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs new file mode 100644 index 0000000000..86c18d0e1a --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class EventLogItems : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "EventLogItems", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + event_log_item_id = table.Column(type: "character varying(600)", maxLength: 600, nullable: false), + description = table.Column(type: "text", nullable: false), + severity = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), + raised_at = table.Column(type: "timestamp with time zone", nullable: false), + related_to = table.Column>(type: "text[]", nullable: false), + category = table.Column(type: "character varying(255)", maxLength: 255, nullable: false), + event_type = table.Column(type: "character varying(255)", maxLength: 255, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_event_log_items", x => x.id); + }); + + migrationBuilder.CreateIndex( + name: "ix_event_log_items_raised_at_id", + table: "EventLogItems", + columns: new[] { "raised_at", "id" }, + descending: new bool[0]); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "EventLogItems"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 770e5515d5..9300d6655e 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -1,5 +1,6 @@ // using System; +using System.Collections.Generic; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; @@ -39,6 +40,63 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("endpoint_settings", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("character varying(600)") + .HasColumnName("event_log_item_id"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("EventLogItems", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => { b.Property("UniqueMessageId") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs deleted file mode 100644 index f7495d6209..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.Designer.cs +++ /dev/null @@ -1,130 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using ServiceControl.Persistence.EFCore.SqlServer; - -#nullable disable - -namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations -{ - [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260722081605_EventLogItems")] - partial class EventLogItems - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 128); - - SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint"); - - SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); - - b.Property("Category") - .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); - - b.Property("Description") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("nvarchar(600)"); - - b.Property("EventType") - .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); - - b.Property("RaisedAt") - .HasColumnType("datetime2"); - - b.PrimitiveCollection("RelatedTo") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("Severity") - .IsRequired() - .HasMaxLength(20) - .HasColumnType("nvarchar(20)"); - - b.HasKey("Id"); - - b.HasIndex("RaisedAt", "Id") - .IsDescending(); - - b.ToTable("EventLogItems", (string)null); - }); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => - { - b.Property("Id") - .HasColumnType("uniqueidentifier"); - - b.Property("Host") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("HostId") - .HasColumnType("uniqueidentifier"); - - b.Property("Monitored") - .HasColumnType("bit"); - - b.Property("Name") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.HasKey("Id"); - - b.ToTable("KnownEndpoints", (string)null); - }); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint"); - - SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); - - b.Property("Host") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("HostId") - .HasColumnType("uniqueidentifier"); - - b.Property("KnownEndpointId") - .HasColumnType("uniqueidentifier"); - - b.Property("Name") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.HasKey("Id"); - - b.HasIndex("KnownEndpointId"); - - b.ToTable("KnownEndpointsInsertOnly", (string)null); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs new file mode 100644 index 0000000000..39cb2c64cc --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs @@ -0,0 +1,261 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260727055119_EventLogItems")] + partial class EventLogItems + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("nvarchar(600)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("QueueAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs similarity index 97% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs index 7c3dbacbc8..0262e57ed4 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722081605_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs @@ -19,7 +19,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("SqlServer:Identity", "1, 1"), EventLogItemId = table.Column(type: "nvarchar(600)", maxLength: 600, nullable: false), Description = table.Column(type: "nvarchar(max)", nullable: false), - Severity = table.Column(type: "nvarchar(20)", maxLength: 20, nullable: false), + Severity = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), RaisedAt = table.Column(type: "datetime2", nullable: false), RelatedTo = table.Column(type: "nvarchar(max)", nullable: false), Category = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index 7fc280a0d8..a7de02a98e 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -71,6 +71,53 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("FailedErrorImports"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventLogItemId") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("nvarchar(600)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { b.Property("UniqueMessageId") @@ -227,53 +274,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("FailedMessageRetries"); }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint"); - - SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); - - b.Property("Category") - .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); - - b.Property("Description") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("nvarchar(600)"); - - b.Property("EventType") - .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); - - b.Property("RaisedAt") - .HasColumnType("datetime2"); - - b.PrimitiveCollection("RelatedTo") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("Severity") - .IsRequired() - .HasMaxLength(20) - .HasColumnType("nvarchar(20)"); - - b.HasKey("Id"); - - b.HasIndex("RaisedAt", "Id") - .IsDescending(); - - b.ToTable("EventLogItems", (string)null); - }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs index a6a39cbf01..519a1f6417 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs @@ -15,14 +15,15 @@ public void Configure(EntityTypeBuilder builder) // indexed and not unique. Nothing looks an item up by it, and each id embeds a fresh Guid. builder.Property(e => e.EventLogItemId).IsRequired().HasMaxLength(600); builder.Property(e => e.Description).IsRequired(); - builder.Property(e => e.Severity).IsRequired().HasConversion().HasMaxLength(20); + builder.Property(e => e.Severity).IsRequired().HasConversion().HasMaxLength(50); builder.Property(e => e.RaisedAt).IsRequired(); // RelatedTo is only ever returned to the API, never queried, so it does not warrant a child table. builder.Property(e => e.RelatedTo).IsRequired(); builder.Property(e => e.Category).IsRequired().HasMaxLength(255); builder.Property(e => e.EventType).IsRequired().HasMaxLength(255); // Every read is "order by RaisedAt descending" plus paging. The key is included as a - // tiebreaker so that items sharing a RaisedAt do not shuffle between pages. + // tiebreaker so that items sharing a RaisedAt do not shuffle between pages, and so that + // MAX(RaisedAt) for the ETag is an index seek. builder.HasIndex(e => new { e.RaisedAt, e.Id }).IsDescending(); } } From 34f50d49f5fd0cd9eff8a06875204e9f6894d768 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 27 Jul 2026 17:21:00 +0800 Subject: [PATCH 03/15] Implement EventLogDataStore methods for adding and retrieving event log items with versioning support --- docs/testing.md | 22 ++++ ...en_the_event_log_is_polled_with_an_etag.cs | 104 ++++++++++++++++++ .../Implementation/EventLogDataStore.cs | 69 +++++++++++- .../EventLogDataStore.cs | 10 +- .../EventLogDataStoreTests.cs | 57 ++++++++++ .../IEventLogDataStore.cs | 22 +++- 6 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs diff --git a/docs/testing.md b/docs/testing.md index e241c6b361..e0ed277c96 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -33,6 +33,28 @@ Transport tests are done by executing the transport test suite for each transpor Run ServiceControl full version and use the HTTP API to validate results. LearningTransport is used for all tests. +### Windows prerequisite: register the event sources + +On Windows, every acceptance test fails on first run with: + +``` +System.Security.SecurityException : The source ServiceControl was not found on computer ., +but some or all event logs could not be searched. Inaccessible logs: Security. +``` + +Setup registers a Windows event source, and checking whether one already exists enumerates every event log — including `Security`, which a process without administrator rights cannot read. + +Register both sources once, from an **elevated** PowerShell prompt: + +```powershell +[System.Diagnostics.EventLog]::CreateEventSource('ServiceControl', 'Application') +[System.Diagnostics.EventLog]::CreateEventSource('ServiceControl.Audit', 'Application') +``` + +Use the .NET API as shown rather than `New-EventLog`, which is not available in PowerShell 7. Afterwards the tests run normally without elevation, because the lookup finds the registered source before it needs to read `Security`. + +Despite mentioning "event logs", this is unrelated to ServiceControl's own event log feature. + ## Multi-instance tests Multi-instance tests validate the interaction between different ServiceControl instances. ServiceControl instances are run in-memory in the same process. LearningTransport is used for all tests. diff --git a/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs b/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs new file mode 100644 index 0000000000..b3c758bfef --- /dev/null +++ b/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs @@ -0,0 +1,104 @@ +namespace ServiceControl.AcceptanceTests.EventLogs +{ + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Net.Http.Json; + using System.Threading.Tasks; + using AcceptanceTesting; + using AcceptanceTesting.EndpointTemplates; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.EventLog; + + [TestFixture] + class When_the_event_log_is_polled_with_an_etag : AcceptanceTest + { + [Test] + public async Task Should_answer_not_modified_only_for_the_current_etag() + { + string etag = null; + HttpStatusCode currentEtagStatus = default; + HttpStatusCode unknownEtagStatus = default; + var unknownEtagReturnedItems = false; + + await Define() + .WithEndpoint() + .Done(async c => + { + var first = await this.GetRaw("/api/eventlogitems/"); + + if (first.StatusCode != HttpStatusCode.OK) + { + return false; + } + + var items = await first.Content.ReadFromJsonAsync>(SerializerOptions); + + // Keep polling until the endpoint's startup event has landed. An empty event + // log has a stable ETag of its own and would make the assertions meaningless. + if (items is not { Count: > 0 }) + { + return false; + } + + // Raw header: an unquoted value fails EntityTagHeaderValue parsing, and this test + // has to observe that rather than throw on it. + if (!first.Headers.TryGetValues("ETag", out var values)) + { + return false; + } + + etag = values.FirstOrDefault(); + + if (string.IsNullOrEmpty(etag)) + { + return false; + } + + currentEtagStatus = (await Poll(etag)).StatusCode; + + var unknown = await Poll("\"not-an-etag-this-instance-ever-issued\""); + unknownEtagStatus = unknown.StatusCode; + + var unknownBody = await unknown.Content.ReadFromJsonAsync>(SerializerOptions); + unknownEtagReturnedItems = unknownBody is { Count: > 0 }; + + return true; + }) + .Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(etag, Is.Not.Null.And.Not.Empty, + "the endpoint must emit an ETag or nothing downstream can cache it"); + + Assert.That(currentEtagStatus, Is.EqualTo(HttpStatusCode.NotModified), + "a client holding the current version must be told so, not handed the page again"); + + Assert.That(unknownEtagStatus, Is.EqualTo(HttpStatusCode.OK), + "an unrecognised validator is a cache miss — this is what stops an unconditional 304 passing"); + + Assert.That(unknownEtagReturnedItems, Is.True, + "a cache miss must carry the items, not an empty body with a 200"); + } + } + + Task Poll(string ifNoneMatch) + { + var request = new HttpRequestMessage(HttpMethod.Get, "/api/eventlogitems/"); + request.Headers.TryAddWithoutValidation("If-None-Match", ifNoneMatch); + + return HttpClient.SendAsync(request); + } + + public class StartingEndpoint : EndpointConfigurationBuilder + { + public StartingEndpoint() => + EndpointSetup(c => c.SendHeartbeatTo(Settings.DEFAULT_INSTANCE_NAME)); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index 1b9799b989..b9f2f53fdf 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -1,13 +1,74 @@ namespace ServiceControl.Persistence.EFCore.Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using ServiceControl.EventLog; using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.EFCore.Entities; -public class EventLogDataStore : IEventLogDataStore +public class EventLogDataStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IEventLogDataStore { public Task Add(EventLogItem logItem) => - throw new NotImplementedException(); + ExecuteWithDbContext(async dbContext => + { + dbContext.EventLogItems.Add(new EventLogItemEntity + { + EventLogItemId = logItem.Id, + Description = logItem.Description, + Severity = logItem.Severity, + RaisedAt = logItem.RaisedAt, + // The column is non-null. The API model allows null and the SignalR broadcast path + // deliberately empties it, so normalise rather than reject. + RelatedTo = logItem.RelatedTo ?? [], + Category = logItem.Category, + EventType = logItem.EventType + }); - public Task<(IList items, long total, string version)> GetEventLogItems(PagingInfo pagingInfo) => - throw new NotImplementedException(); + await dbContext.SaveChangesAsync(); + }); + + public Task<(IList? items, long total, string version)> GetEventLogItems( + PagingInfo pagingInfo, string? knownVersion = null) => + ExecuteWithDbContext<(IList? items, long total, string version)>(async dbContext => + { + var query = dbContext.EventLogItems.AsNoTracking(); + + var total = await query.LongCountAsync(); + var newest = await query.MaxAsync(e => (DateTime?)e.RaisedAt); + var version = Version(total, newest); + + // The point of knownVersion. Everything above is index work. + // If the caller already has the latest version, skip the rest of the query. + // No database round trip is needed. No response body is needed. + if (knownVersion is not null && knownVersion == version) + { + return (null, total, version); + } + + var items = await query + // The key breaks ties so that items sharing a RaisedAt cannot shuffle between + // pages. IX_EventLogItems_RaisedAt_Id is declared in exactly this order. + .OrderByDescending(e => e.RaisedAt) + .ThenByDescending(e => e.Id) + .Skip(pagingInfo.Offset) + .Take(pagingInfo.PageSize) + .Select(e => new EventLogItem + { + Id = e.EventLogItemId, + Description = e.Description, + Severity = e.Severity, + RaisedAt = e.RaisedAt, + RelatedTo = e.RelatedTo, + Category = e.Category, + EventType = e.EventType + }) + .ToListAsync(); + + return (items, total, version); + }); + + // Raven returns the query's ResultEtag but there is no relational equivalent, so it is synthesised + // using total count and the newest item's RaisedAt timestamp. + static string Version(long total, DateTime? newest) => + DeterministicGuid.MakeId($"{total}|{newest?.Ticks ?? 0}").ToString(); } diff --git a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs index d3e2f5d850..ef51444a15 100644 --- a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs @@ -18,7 +18,8 @@ public async Task Add(EventLogItem logItem) await session.SaveChangesAsync(); } - public async Task<(IList, long, string)> GetEventLogItems(PagingInfo pagingInfo) + public async Task<(IList, long, string)> GetEventLogItems( + PagingInfo pagingInfo, string knownVersion = null) { using var session = await sessionProvider.OpenSession(); var results = await session @@ -28,7 +29,12 @@ public async Task Add(EventLogItem logItem) .Paging(pagingInfo) .ToListAsync(); - return (results, stats.TotalResults, stats.ResultEtag.ToString()); + var version = stats.ResultEtag.ToString(); + + // For robustness and consistency. Decide 304s at the controller level. + var unchanged = knownVersion is not null && knownVersion == version; + + return (unchanged ? null : results, stats.TotalResults, version); } } } diff --git a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs index 85488a541b..4d04c4c1c8 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs @@ -157,6 +157,63 @@ public async Task Version_is_stable_while_nothing_changes() Assert.That(secondRead, Is.EqualTo(firstRead)); } + [Test] + public async Task Matching_known_version_reports_no_items() + { + await AddItems(3); + var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); + + Assert.That(items, Is.Null, "a caller already holding the current version must be told so, not handed the page again"); + } + + [Test] + public async Task Matching_known_version_still_reports_total_and_version() + { + await AddItems(3); + var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + var (_, total, versionAgain) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); + + using (Assert.EnterMultipleScope()) + { + // The controller sets Total-Count and ETag on the 304, so neither may be dropped + // just because the page was not fetched. + Assert.That(total, Is.EqualTo(3)); + Assert.That(versionAgain, Is.EqualTo(version)); + } + } + + [Test] + public async Task Stale_known_version_returns_the_page() + { + await AddItems(2); + var (_, _, staleVersion) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + await AddItems(1); + + var (items, total, freshVersion) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), staleVersion); + + using (Assert.EnterMultipleScope()) + { + Assert.That(items, Is.Not.Null); + Assert.That(items, Has.Count.EqualTo(3)); + Assert.That(total, Is.EqualTo(3)); + Assert.That(freshVersion, Is.Not.EqualTo(staleVersion)); + } + } + + [Test] + public async Task Unrecognised_known_version_returns_the_page() + { + await AddItems(2); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), "not-a-version-this-store-ever-issued"); + + Assert.That(items, Is.Not.Null, "an unrecognised validator must be treated as a cache miss, never as a match"); + } + async Task AddItems(int count) { var baseTime = new DateTime(2026, 7, 22, 8, 0, 0, DateTimeKind.Utc); diff --git a/src/ServiceControl.Persistence/IEventLogDataStore.cs b/src/ServiceControl.Persistence/IEventLogDataStore.cs index 33d71b7a68..d43d4040f3 100644 --- a/src/ServiceControl.Persistence/IEventLogDataStore.cs +++ b/src/ServiceControl.Persistence/IEventLogDataStore.cs @@ -7,7 +7,27 @@ public interface IEventLogDataStore { + /// + /// Persists a single event log item. arrives fully formed from + /// EventLogMappingDefinition.Apply, including its Id, which a persister must + /// treat as opaque. + /// Task Add(EventLogItem logItem); - Task<(IList items, long total, string version)> GetEventLogItems(PagingInfo pagingInfo); + + /// + /// Returns one page of event log items, newest RaisedAt first. + /// + /// Which page to return. + /// + /// The version the caller already holds, or null if it holds none. + /// + /// + /// items — the requested page, which may be empty; + /// total — the number of items in the store, independent of the page size; + /// version — an opaque cache validator surfaced as the ETag response header by + /// EventLogApiController. + /// + Task<(IList items, long total, string version)> GetEventLogItems( + PagingInfo pagingInfo, string knownVersion = null); } } \ No newline at end of file From 53ec9b93048968475d42deca22c8e8c4b9235b15 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 27 Jul 2026 22:49:40 +0800 Subject: [PATCH 04/15] Add event log retention period and implement sweeping logic for expired items --- .../EFPersistenceConfigurationBase.cs | 2 + .../Abstractions/EFPersisterSettings.cs | 2 + .../Infrastructure/RetentionSweeper.cs | 60 ++++++++- .../EventLogDataStore.cs | 2 + .../EFCore/EventLogDataStoreEFTests.cs | 123 ++++++++++++++++++ ...EventLogKnownVersionSkipsPageQueryTests.cs | 94 +++++++++++++ .../EFCore/RetentionSweepTests.cs | 76 +++++++++++ .../EventLogWriterTests.cs | 61 +++++++++ .../EventLog/AuditEventLogWriter.cs | 4 +- .../EventLog/EventLogApiController.cs | 19 ++- 10 files changed, 432 insertions(+), 11 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs index 4d752ad296..eb936c24bb 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs @@ -25,6 +25,7 @@ public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize"; const string MaxBodySizeToStoreKey = "MaxBodySizeToStore"; const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod"; + const string EventsRetentionPeriodKey = "EventsRetentionPeriod"; const string EnableFullTextSearchOnBodiesKey = "EnableFullTextSearchOnBodies"; const string SubscriptionCacheDurationKey = "SubscriptionCacheDuration"; @@ -36,6 +37,7 @@ public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootName settings.CommandTimeout = SettingsReader.Read(settingsRootNamespace, CommandTimeoutKey, EFPersisterSettings.DefaultCommandTimeout); settings.ErrorRetentionPeriod = GetRequiredSetting(settingsRootNamespace, ErrorRetentionPeriodKey); + settings.EventsRetentionPeriod = SettingsReader.Read(settingsRootNamespace, EventsRetentionPeriodKey, EFPersisterSettings.DefaultEventsRetentionPeriod); settings.EnableFullTextSearchOnBodies = SettingsReader.Read(settingsRootNamespace, EnableFullTextSearchOnBodiesKey, true); settings.SubscriptionCacheDuration = SettingsReader.Read(settingsRootNamespace, SubscriptionCacheDurationKey, EFPersisterSettings.DefaultSubscriptionCacheDuration); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs index 5eed5da073..7744f65dc0 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs @@ -5,12 +5,14 @@ public abstract class EFPersisterSettings : PersistenceSettings public static readonly TimeSpan MigrationCommandTimeout = TimeSpan.FromMinutes(40); public const int DefaultCommandTimeout = 30; + public static readonly TimeSpan DefaultEventsRetentionPeriod = TimeSpan.FromDays(14); public static readonly TimeSpan DefaultSubscriptionCacheDuration = TimeSpan.FromSeconds(60); public required string ConnectionString { get; set; } public int CommandTimeout { get; set; } = DefaultCommandTimeout; public TimeSpan ErrorRetentionPeriod { get; set; } + public TimeSpan EventsRetentionPeriod { get; set; } = DefaultEventsRetentionPeriod; public required BodyStorageSettings BodyStorage { get; set; } public int MaxRetryCount { get; set; } = 5; public int MaxRetryDelayInSeconds { get; set; } = 30; diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs index 8f5d2d165d..f598cae42b 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs @@ -9,9 +9,11 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; -// Deletes resolved and archived failed messages once they age past the retention period. Runs -// hourly, in bounded batches so it never holds a large delete, and recomputes the cutoff on every -// run so a changed retention setting takes effect without rewriting any row. +// Deletes rows once they age past their retention period. +// Runs hourly, in bounded batches so it never holds a large delete, and recomputes the cutoffs on +// every run so a changed retention setting takes effect without rewriting any row. +// +// RavenDB has no equivalent: it stamps per-document expiry metadata at write time instead. public class RetentionSweeper( ILogger logger, TimeProvider timeProvider, @@ -26,7 +28,7 @@ public class RetentionSweeper( protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - logger.LogInformation("Starting error retention sweep"); + logger.LogInformation("Starting retention sweep"); try { @@ -42,13 +44,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } catch (Exception ex) when (ex is not OperationCanceledException) { - logger.LogError(ex, "Error during error retention sweep"); + logger.LogError(ex, "Error during retention sweep"); } } while (await timer.WaitForNextTickAsync(stoppingToken)); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - logger.LogInformation("Stopping error retention sweep"); + logger.LogInformation("Stopping retention sweep"); } } @@ -57,6 +59,52 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) public Task SweepNow(CancellationToken cancellationToken = default) => Sweep(pace: false, cancellationToken); async Task Sweep(bool pace, CancellationToken cancellationToken) + { + await SweepFailedMessages(pace, cancellationToken); + await SweepEventLogItems(pace, cancellationToken); + } + + // Event log items are insert-only and carry no external bodies, so this is a straight batched + // delete by age, no per-row work, no re-asserted predicate, nothing to clean up first. + async Task SweepEventLogItems(bool pace, CancellationToken cancellationToken) + { + var cutoff = timeProvider.GetUtcNow().UtcDateTime - settings.EventsRetentionPeriod; + + while (!cancellationToken.IsCancellationRequested) + { + using var scope = serviceScopeFactory.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var expired = await dbContext.EventLogItems + .AsNoTracking() + .Where(eventLogItem => eventLogItem.RaisedAt < cutoff) + .OrderBy(eventLogItem => eventLogItem.RaisedAt) + .Take(BatchSize) + .Select(eventLogItem => eventLogItem.Id) + .ToListAsync(cancellationToken); + + if (expired.Count == 0) + { + break; + } + + await dbContext.EventLogItems + .Where(eventLogItem => expired.Contains(eventLogItem.Id)) + .ExecuteDeleteAsync(cancellationToken); + + if (expired.Count < BatchSize) + { + break; + } + + if (pace) + { + await Task.Delay(BatchPause, timeProvider, cancellationToken); + } + } + } + + async Task SweepFailedMessages(bool pace, CancellationToken cancellationToken) { var cutoff = timeProvider.GetUtcNow().UtcDateTime - settings.ErrorRetentionPeriod; diff --git a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs index ef51444a15..b6d7abbbcb 100644 --- a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs @@ -13,6 +13,8 @@ public async Task Add(EventLogItem logItem) using var session = await sessionProvider.OpenSession(); await session.StoreAsync(logItem); + // Retention on RavenDB is per-document expiry metadata stamped at write time, not a + // sweep. It has to be set here, on the only write path, or items never expire. expirationManager.EnableExpiration(session, logItem); await session.SaveChangesAsync(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs new file mode 100644 index 0000000000..3405e53b93 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -0,0 +1,123 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.EventLog; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.Infrastructure; + +// Behaviour only the EF Core persisters guarantee, so it cannot live in the shared root folder. +// RavenDB orders event log items by RaisedAt alone. +class EventLogDataStoreEFTests : PersistenceTestBase +{ + // All ten items share one RaisedAt. Without the key as a tiebreaker in both the index and the + // ORDER BY, the engine is free to return them in a different order per query, and items would + // silently appear on two pages, or on none. + [Test] + public async Task Items_sharing_a_raised_at_page_without_overlap_or_omission() + { + var sameInstant = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc); + var expected = new List(); + + for (var i = 0; i < 10; i++) + { + var item = CreateLogItem($"Collision{i}", sameInstant); + expected.Add(item.Id); + await EventLogDataStore.Add(item); + } + + await CompleteDatabaseOperation(); + + var paged = new List(); + for (var page = 1; page <= 5; page++) + { + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: page, pageSize: 2)); + paged.AddRange(items.Select(i => i.Id)); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(paged, Has.Count.EqualTo(10), "every item must appear exactly once across the pages"); + Assert.That(paged.Distinct().Count(), Is.EqualTo(10), "no item may appear on two pages"); + Assert.That(paged, Is.EquivalentTo(expected), "no item may be omitted"); + } + } + + [Test] + public async Task Paging_order_is_repeatable_when_raised_at_collides() + { + var sameInstant = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc); + + for (var i = 0; i < 6; i++) + { + await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant)); + } + + await CompleteDatabaseOperation(); + + var (first, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6)); + var (second, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6)); + + Assert.That(second.Select(i => i.Id), Is.EqualTo(first.Select(i => i.Id)).AsCollection); + } + + // The count term in the version exists so that retention deleting the oldest rows invalidates a + // client's cache. Nothing else verifies that, because IEventLogDataStore has no delete. + [Test] + public async Task Version_changes_when_rows_are_deleted_behind_the_interface() + { + var baseTime = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc); + + for (var i = 0; i < 3; i++) + { + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); + } + + await CompleteDatabaseOperation(); + + var (_, totalBefore, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + Assert.That(totalBefore, Is.EqualTo(3)); + + // Delete the OLDEST row — the one a retention sweep would take. A rowversion or MAX(RaisedAt) + // scheme would be entirely blind to this; only the count term catches it. + await DeleteOldest(); + + var (_, totalAfter, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(totalAfter, Is.EqualTo(2)); + Assert.That(versionAfter, Is.Not.EqualTo(versionBefore), "a delete must invalidate the client's cached page"); + } + } + + async Task DeleteOldest() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var oldest = await dbContext.EventLogItems + .OrderBy(e => e.RaisedAt) + .ThenBy(e => e.Id) + .FirstAsync(TestContext.CurrentContext.CancellationToken); + + dbContext.EventLogItems.Remove(oldest); + await dbContext.SaveChangesAsync(TestContext.CurrentContext.CancellationToken); + } + + static EventLogItem CreateLogItem(string eventType, DateTime raisedAt) => new() + { + Id = $"EventLogItem/Recoverability/{eventType}/{Guid.NewGuid()}", + Category = "Recoverability", + EventType = eventType, + Description = $"{eventType} occurred", + Severity = Severity.Info, + RaisedAt = raisedAt, + RelatedTo = [] + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs new file mode 100644 index 0000000000..3907c68fd1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using ServiceControl.EventLog; +using ServiceControl.Persistence.Infrastructure; + +// EF only. RavenDB gets documents, total and etag from a single query, so it has nothing to skip +class EventLogKnownVersionSkipsPageQueryTests : PersistenceTestBase +{ + readonly CommandCounter counter = new(); + + public EventLogKnownVersionSkipsPageQueryTests() => + RegisterServices = services => services.AddSingleton(counter); + + [Test] + public async Task A_matching_known_version_does_not_run_the_page_query() + { + await EventLogDataStore.Add(CreateLogItem()); + await CompleteDatabaseOperation(); + + var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + counter.Reset(); + await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); + var whenCurrent = counter.Count; + + counter.Reset(); + await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var whenFetching = counter.Count; + + using (Assert.EnterMultipleScope()) + { + Assert.That(whenCurrent, Is.EqualTo(2), "count and max only — the page query must not run"); + Assert.That(whenFetching, Is.EqualTo(3), "count, max and the page query"); + } + } + + static EventLogItem CreateLogItem() => new() + { + Id = $"EventLogItem/Recoverability/MessageFailed/{Guid.NewGuid()}", + Category = "Recoverability", + EventType = "MessageFailed", + Description = "failed", + Severity = Severity.Info, + RaisedAt = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc), + RelatedTo = [] + }; + + sealed class CommandCounter : ILoggerProvider + { + int count; + + public int Count => Volatile.Read(ref count); + + public void Reset() => Volatile.Write(ref count, 0); + + // Only the relational command category matters; everything else the host logs is ignored. + public ILogger CreateLogger(string categoryName) => + categoryName == DbLoggerCategory.Database.Command.Name + ? new CountingLogger(this) + : NullLogger.Instance; + + public void Dispose() + { + } + + sealed class CountingLogger(CommandCounter owner) : ILogger + { + public IDisposable BeginScope(TState state) where TState : notnull => + NullLogger.Instance.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, + Func formatter) + { + // CommandExecuted fires once per completed command, so it counts round trips rather + // than retries or connection events. + if (eventId == RelationalEventId.CommandExecuted) + { + Interlocked.Increment(ref owner.count); + } + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index eb2337e595..e14a31c46d 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -1,10 +1,14 @@ namespace ServiceControl.Persistence.Tests; using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using NUnit.Framework; +using ServiceControl.EventLog; using ServiceControl.MessageFailures; using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.Infrastructure; class RetentionSweepTests : ErrorIngestionTestBase { @@ -140,4 +144,76 @@ await Store(new FailedMessageEntity return id; } + + [Test] + public async Task Deletes_event_log_items_past_the_events_cutoff() + { + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); + + await Store(EventLogRow("expired", Now.AddDays(-15))); + await Store(EventLogRow("fresh", Now.AddDays(-13))); + + await RunRetentionSweep(); + + var remaining = await GetEventLogItemIds(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(remaining, Does.Not.Contain("expired")); + Assert.That(remaining, Does.Contain("fresh")); + } + } + + [Test] + public async Task Event_log_retention_is_independent() + { + // A 30 day error retention must not keep a 1 day event log item alive. + EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(30); + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(1); + + await Store(EventLogRow("old-event", Now.AddDays(-2))); + + await RunRetentionSweep(); + + Assert.That(await GetEventLogItemIds(), Does.Not.Contain("old-event")); + } + + [Test] + public async Task Sweeping_event_log_items_changes_the_version() + { + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); + + await Store(EventLogRow("expired", Now.AddDays(-15))); + await Store(EventLogRow("fresh", Now.AddDays(-1))); + + var (_, _, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + await RunRetentionSweep(); + + var (_, total, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(total, Is.EqualTo(1)); + // The count term of the version exists precisely so that retention invalidates client caches. + Assert.That(versionAfter, Is.Not.EqualTo(versionBefore)); + } + } + + static EventLogItemEntity EventLogRow(string id, DateTime raisedAt) => new() + { + EventLogItemId = id, + Description = "swept", + Severity = Severity.Info, + RaisedAt = raisedAt, + RelatedTo = [], + Category = "Recoverability", + EventType = "MessageFailed" + }; + + async Task> GetEventLogItemIds() + { + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 100)); + return [.. items.Select(i => i.Id)]; + } } diff --git a/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs new file mode 100644 index 0000000000..bd603dfc3f --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs @@ -0,0 +1,61 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.EventLog; +using ServiceControl.Infrastructure.DomainEvents; +using ServiceControl.Persistence.Infrastructure; + +// This exercises writer -> store -> row -> readable on every persister, using a test-local domain +// event and mapping rather than borrowing a production contract, so it pins the writer mechanism +// rather than the shape of MessageFailed. +class EventLogWriterTests : PersistenceTestBase +{ + [Test] + public async Task A_mapped_domain_event_is_persisted_and_can_be_read_back() + { + var writer = CreateWriter(); + + await writer.Handle(new SomethingHappened { What = "it happened" }, CancellationToken.None); + await CompleteDatabaseOperation(); + + var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(total, Is.EqualTo(1)); + Assert.That(items.Single().Description, Is.EqualTo("it happened")); + Assert.That(items.Single().EventType, Is.EqualTo(nameof(SomethingHappened))); + } + } + + [Test] + public async Task An_unmapped_domain_event_is_ignored() + { + var writer = CreateWriter(); + + await writer.Handle(new NothingMapsThis(), CancellationToken.None); + await CompleteDatabaseOperation(); + + var (_, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(total, Is.Zero, "only events with a mapping under EventLog\\Definitions are recorded"); + } + + AuditEventLogWriter CreateWriter() => + new(EventLogDataStore, new EventLogMappings([new SomethingHappenedDefinition()])); + + class SomethingHappened : IDomainEvent + { + public string What { get; set; } + } + + class NothingMapsThis : IDomainEvent; + + class SomethingHappenedDefinition : EventLogMappingDefinition + { + public SomethingHappenedDefinition() => Description(m => m.What); + } +} diff --git a/src/ServiceControl/EventLog/AuditEventLogWriter.cs b/src/ServiceControl/EventLog/AuditEventLogWriter.cs index af51a66135..6e83d68b81 100644 --- a/src/ServiceControl/EventLog/AuditEventLogWriter.cs +++ b/src/ServiceControl/EventLog/AuditEventLogWriter.cs @@ -6,8 +6,8 @@ namespace ServiceControl.EventLog using ServiceControl.Persistence; /// - /// Only for events that have been defined (under EventLog\Definitions), a logentry item will - /// be saved in Raven. + /// Only for events that have been defined (under EventLog\Definitions), a log entry item is + /// persisted. /// class AuditEventLogWriter : IDomainHandler { diff --git a/src/ServiceControl/EventLog/EventLogApiController.cs b/src/ServiceControl/EventLog/EventLogApiController.cs index 605efc453d..253e2275fd 100644 --- a/src/ServiceControl/EventLog/EventLogApiController.cs +++ b/src/ServiceControl/EventLog/EventLogApiController.cs @@ -1,6 +1,8 @@ namespace ServiceControl.EventLog { using System.Collections.Generic; + using System.Linq; + using System.Net; using System.Threading.Tasks; using Infrastructure.Auth; using Infrastructure.WebApi; @@ -16,14 +18,25 @@ public class EventLogApiController(IEventLogDataStore logDataStore) : Controller [Authorize(Policy = Permissions.ErrorEventLogView)] [Route("eventlogitems")] [HttpGet] - public async Task> Items([FromQuery] PagingInfo pagingInfo) + public async Task>> Items([FromQuery] PagingInfo pagingInfo) { - var (results, totalCount, version) = await logDataStore.GetEventLogItems(pagingInfo); + + // The Trim handles both ETag formats (quoted and unquoted) deliberately. + var knownVersion = Request.Headers.IfNoneMatch.FirstOrDefault()?.Trim('"'); + + // Passing knownVersion lets the persister skip work it would otherwise waste + var (results, totalCount, version) = await logDataStore.GetEventLogItems(pagingInfo, knownVersion); Response.WithPagingLinksAndTotalCount(pagingInfo, totalCount); Response.WithEtag(version); - return results; + // Null items means "you already hold this version" + if (results is null) + { + return StatusCode((int)HttpStatusCode.NotModified); + } + + return Ok(results); } } } \ No newline at end of file From 2b63efdd45d54e40027a0ad5793355ea942b2eb9 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 27 Jul 2026 23:08:57 +0800 Subject: [PATCH 05/15] Enhance documentation for IEventLogDataStore interface, clarifying item retention and cache validation behavior --- src/ServiceControl.Persistence/IEventLogDataStore.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence/IEventLogDataStore.cs b/src/ServiceControl.Persistence/IEventLogDataStore.cs index d43d4040f3..3e6a329a07 100644 --- a/src/ServiceControl.Persistence/IEventLogDataStore.cs +++ b/src/ServiceControl.Persistence/IEventLogDataStore.cs @@ -5,6 +5,14 @@ using EventLog; using Infrastructure; + /// + /// Stores and reads the event log. + /// + /// Items are subject to provider retention and disappear on their own. Each persister + /// enforces this its own way and neither is visible here. RavenDB stamps per-document expiry + /// metadata when the item is written, EF Core deletes aged rows on a timer. + /// + /// public interface IEventLogDataStore { /// @@ -25,7 +33,8 @@ public interface IEventLogDataStore /// items — the requested page, which may be empty; /// total — the number of items in the store, independent of the page size; /// version — an opaque cache validator surfaced as the ETag response header by - /// EventLogApiController. + /// EventLogApiController. It must change when retention removes items, not only when + /// one is added, since nothing else tells a client its cached page is now wrong. /// Task<(IList items, long total, string version)> GetEventLogItems( PagingInfo pagingInfo, string knownVersion = null); From 6ba67172f16a852b51f2d2a526782255d3e131a4 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 28 Jul 2026 08:25:07 +0800 Subject: [PATCH 06/15] Add EnableIntegratedServicePulse setting to App.config --- src/ServiceControl/App.config | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl/App.config b/src/ServiceControl/App.config index cf671edde0..4a77515a3b 100644 --- a/src/ServiceControl/App.config +++ b/src/ServiceControl/App.config @@ -11,6 +11,7 @@ These settings are only here so that we can debug ServiceControl while developin + From 0b2cb5f88998504c7dbc6e7f79a96acef6919691 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 28 Jul 2026 16:37:28 +0800 Subject: [PATCH 07/15] Review changes --- docs/testing.md | 2 - ... 20260728075200_EventLogItems.Designer.cs} | 8 +- ...ems.cs => 20260728075200_EventLogItems.cs} | 2 +- ...SqlServiceControlDbContextModelSnapshot.cs | 6 +- ... 20260728075138_EventLogItems.Designer.cs} | 8 +- ...ems.cs => 20260728075138_EventLogItems.cs} | 2 +- ...verServiceControlDbContextModelSnapshot.cs | 6 +- .../EventLogItemConfiguration.cs | 2 +- .../Implementation/EventLogDataStore.cs | 17 ++-- .../Infrastructure/RetentionSweeper.cs | 21 +---- .../EFCore/EventLogDataStoreEFTests.cs | 2 - ...EventLogKnownVersionSkipsPageQueryTests.cs | 94 ------------------- .../EventLogWriterTests.cs | 3 - .../EventLog/AuditEventLogWriter.cs | 4 +- 14 files changed, 29 insertions(+), 148 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260727055524_EventLogItems.Designer.cs => 20260728075200_EventLogItems.Designer.cs} (98%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260727055524_EventLogItems.cs => 20260728075200_EventLogItems.cs} (94%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260727055119_EventLogItems.Designer.cs => 20260728075138_EventLogItems.Designer.cs} (97%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260727055119_EventLogItems.cs => 20260728075138_EventLogItems.cs} (94%) delete mode 100644 src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs diff --git a/docs/testing.md b/docs/testing.md index e0ed277c96..8ded9542b5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -53,8 +53,6 @@ Register both sources once, from an **elevated** PowerShell prompt: Use the .NET API as shown rather than `New-EventLog`, which is not available in PowerShell 7. Afterwards the tests run normally without elevation, because the lookup finds the registered source before it needs to read `Security`. -Despite mentioning "event logs", this is unrelated to ServiceControl's own event log feature. - ## Multi-instance tests Multi-instance tests validate the interaction between different ServiceControl instances. ServiceControl instances are run in-memory in the same process. LearningTransport is used for all tests. diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs similarity index 98% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs index 4c8c9d4342..9fef432e27 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260727055524_EventLogItems")] + [Migration("20260728075200_EventLogItems")] partial class EventLogItems { /// @@ -67,10 +67,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("text[]") .HasColumnName("related_to"); - b.Property("Severity") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)") + b.Property("Severity") + .HasColumnType("integer") .HasColumnName("severity"); b.HasKey("Id") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.cs similarity index 94% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.cs index 86c18d0e1a..6238f09995 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260727055524_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.cs @@ -21,7 +21,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), event_log_item_id = table.Column(type: "character varying(600)", maxLength: 600, nullable: false), description = table.Column(type: "text", nullable: false), - severity = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), + severity = table.Column(type: "integer", nullable: false), raised_at = table.Column(type: "timestamp with time zone", nullable: false), related_to = table.Column>(type: "text[]", nullable: false), category = table.Column(type: "character varying(255)", maxLength: 255, nullable: false), diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 9300d6655e..cdd95ce052 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -81,10 +81,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("text[]") .HasColumnName("related_to"); - b.Property("Severity") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)") + b.Property("Severity") + .HasColumnType("integer") .HasColumnName("severity"); b.HasKey("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs similarity index 97% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs index 39cb2c64cc..2eb91c6990 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260727055119_EventLogItems")] + [Migration("20260728075138_EventLogItems")] partial class EventLogItems { /// @@ -59,10 +59,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("Severity") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("nvarchar(50)"); + b.Property("Severity") + .HasColumnType("int"); b.HasKey("Id"); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.cs similarity index 94% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.cs index 0262e57ed4..345acc4054 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260727055119_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.cs @@ -19,7 +19,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("SqlServer:Identity", "1, 1"), EventLogItemId = table.Column(type: "nvarchar(600)", maxLength: 600, nullable: false), Description = table.Column(type: "nvarchar(max)", nullable: false), - Severity = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), + Severity = table.Column(type: "int", nullable: false), RaisedAt = table.Column(type: "datetime2", nullable: false), RelatedTo = table.Column(type: "nvarchar(max)", nullable: false), Category = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index a7de02a98e..7f5f923eed 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -105,10 +105,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("Severity") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("nvarchar(50)"); + b.Property("Severity") + .HasColumnType("int"); b.HasKey("Id"); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs index 519a1f6417..8374f9a77b 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs @@ -15,7 +15,7 @@ public void Configure(EntityTypeBuilder builder) // indexed and not unique. Nothing looks an item up by it, and each id embeds a fresh Guid. builder.Property(e => e.EventLogItemId).IsRequired().HasMaxLength(600); builder.Property(e => e.Description).IsRequired(); - builder.Property(e => e.Severity).IsRequired().HasConversion().HasMaxLength(50); + builder.Property(e => e.Severity).IsRequired(); builder.Property(e => e.RaisedAt).IsRequired(); // RelatedTo is only ever returned to the API, never queried, so it does not warrant a child table. builder.Property(e => e.RelatedTo).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index b9f2f53fdf..7c2c2c2669 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -17,8 +17,6 @@ public Task Add(EventLogItem logItem) => Description = logItem.Description, Severity = logItem.Severity, RaisedAt = logItem.RaisedAt, - // The column is non-null. The API model allows null and the SignalR broadcast path - // deliberately empties it, so normalise rather than reject. RelatedTo = logItem.RelatedTo ?? [], Category = logItem.Category, EventType = logItem.EventType @@ -33,9 +31,15 @@ public Task Add(EventLogItem logItem) => { var query = dbContext.EventLogItems.AsNoTracking(); - var total = await query.LongCountAsync(); - var newest = await query.MaxAsync(e => (DateTime?)e.RaisedAt); - var version = Version(total, newest); + // Both aggregates in one round trip. Grouping on a constant collapses the table to a + // single row, and an empty table yields no rows at all, hence the null coalescing. + var stats = await query + .GroupBy(_ => 1) + .Select(g => new { Total = g.LongCount(), Newest = g.Max(e => (DateTime?)e.RaisedAt) }) + .FirstOrDefaultAsync(); + + var total = stats?.Total ?? 0; + var version = Version(total, stats?.Newest); // The point of knownVersion. Everything above is index work. // If the caller already has the latest version, skip the rest of the query. @@ -67,8 +71,7 @@ public Task Add(EventLogItem logItem) => return (items, total, version); }); - // Raven returns the query's ResultEtag but there is no relational equivalent, so it is synthesised - // using total count and the newest item's RaisedAt timestamp. + // Synthesised version ID to be used for an ETag, using total count and the newest item's RaisedAt timestamp. static string Version(long total, DateTime? newest) => DeterministicGuid.MakeId($"{total}|{newest?.Ticks ?? 0}").ToString(); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs index f598cae42b..9fd1cb6b96 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs @@ -12,8 +12,6 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; // Deletes rows once they age past their retention period. // Runs hourly, in bounded batches so it never holds a large delete, and recomputes the cutoffs on // every run so a changed retention setting takes effect without rewriting any row. -// -// RavenDB has no equivalent: it stamps per-document expiry metadata at write time instead. public class RetentionSweeper( ILogger logger, TimeProvider timeProvider, @@ -64,8 +62,8 @@ async Task Sweep(bool pace, CancellationToken cancellationToken) await SweepEventLogItems(pace, cancellationToken); } - // Event log items are insert-only and carry no external bodies, so this is a straight batched - // delete by age, no per-row work, no re-asserted predicate, nothing to clean up first. + // Event log items are insert-only and carry no external bodies, so each batch is a single + // ordered DELETE. async Task SweepEventLogItems(bool pace, CancellationToken cancellationToken) { var cutoff = timeProvider.GetUtcNow().UtcDateTime - settings.EventsRetentionPeriod; @@ -75,24 +73,13 @@ async Task SweepEventLogItems(bool pace, CancellationToken cancellationToken) using var scope = serviceScopeFactory.CreateScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); - var expired = await dbContext.EventLogItems - .AsNoTracking() + var deleted = await dbContext.EventLogItems .Where(eventLogItem => eventLogItem.RaisedAt < cutoff) .OrderBy(eventLogItem => eventLogItem.RaisedAt) .Take(BatchSize) - .Select(eventLogItem => eventLogItem.Id) - .ToListAsync(cancellationToken); - - if (expired.Count == 0) - { - break; - } - - await dbContext.EventLogItems - .Where(eventLogItem => expired.Contains(eventLogItem.Id)) .ExecuteDeleteAsync(cancellationToken); - if (expired.Count < BatchSize) + if (deleted < BatchSize) { break; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs index 3405e53b93..4c7539270d 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -11,8 +11,6 @@ namespace ServiceControl.Persistence.Tests; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.Infrastructure; -// Behaviour only the EF Core persisters guarantee, so it cannot live in the shared root folder. -// RavenDB orders event log items by RaisedAt alone. class EventLogDataStoreEFTests : PersistenceTestBase { // All ten items share one RaisedAt. Without the key as a tiebreaker in both the index and the diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs deleted file mode 100644 index 3907c68fd1..0000000000 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogKnownVersionSkipsPageQueryTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -namespace ServiceControl.Persistence.Tests; - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Diagnostics; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using NUnit.Framework; -using ServiceControl.EventLog; -using ServiceControl.Persistence.Infrastructure; - -// EF only. RavenDB gets documents, total and etag from a single query, so it has nothing to skip -class EventLogKnownVersionSkipsPageQueryTests : PersistenceTestBase -{ - readonly CommandCounter counter = new(); - - public EventLogKnownVersionSkipsPageQueryTests() => - RegisterServices = services => services.AddSingleton(counter); - - [Test] - public async Task A_matching_known_version_does_not_run_the_page_query() - { - await EventLogDataStore.Add(CreateLogItem()); - await CompleteDatabaseOperation(); - - var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); - - counter.Reset(); - await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); - var whenCurrent = counter.Count; - - counter.Reset(); - await EventLogDataStore.GetEventLogItems(new PagingInfo()); - var whenFetching = counter.Count; - - using (Assert.EnterMultipleScope()) - { - Assert.That(whenCurrent, Is.EqualTo(2), "count and max only — the page query must not run"); - Assert.That(whenFetching, Is.EqualTo(3), "count, max and the page query"); - } - } - - static EventLogItem CreateLogItem() => new() - { - Id = $"EventLogItem/Recoverability/MessageFailed/{Guid.NewGuid()}", - Category = "Recoverability", - EventType = "MessageFailed", - Description = "failed", - Severity = Severity.Info, - RaisedAt = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc), - RelatedTo = [] - }; - - sealed class CommandCounter : ILoggerProvider - { - int count; - - public int Count => Volatile.Read(ref count); - - public void Reset() => Volatile.Write(ref count, 0); - - // Only the relational command category matters; everything else the host logs is ignored. - public ILogger CreateLogger(string categoryName) => - categoryName == DbLoggerCategory.Database.Command.Name - ? new CountingLogger(this) - : NullLogger.Instance; - - public void Dispose() - { - } - - sealed class CountingLogger(CommandCounter owner) : ILogger - { - public IDisposable BeginScope(TState state) where TState : notnull => - NullLogger.Instance.BeginScope(state); - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, - Func formatter) - { - // CommandExecuted fires once per completed command, so it counts round trips rather - // than retries or connection events. - if (eventId == RelationalEventId.CommandExecuted) - { - Interlocked.Increment(ref owner.count); - } - } - } - } -} diff --git a/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs index bd603dfc3f..c22c76c93d 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs @@ -8,9 +8,6 @@ namespace ServiceControl.Persistence.Tests; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Persistence.Infrastructure; -// This exercises writer -> store -> row -> readable on every persister, using a test-local domain -// event and mapping rather than borrowing a production contract, so it pins the writer mechanism -// rather than the shape of MessageFailed. class EventLogWriterTests : PersistenceTestBase { [Test] diff --git a/src/ServiceControl/EventLog/AuditEventLogWriter.cs b/src/ServiceControl/EventLog/AuditEventLogWriter.cs index 6e83d68b81..aeb14eafc9 100644 --- a/src/ServiceControl/EventLog/AuditEventLogWriter.cs +++ b/src/ServiceControl/EventLog/AuditEventLogWriter.cs @@ -6,8 +6,8 @@ namespace ServiceControl.EventLog using ServiceControl.Persistence; /// - /// Only for events that have been defined (under EventLog\Definitions), a log entry item is - /// persisted. + /// Only for events with an registered via + /// AddEventLogMapping is a log entry item persisted. /// class AuditEventLogWriter : IDomainHandler { From 8acc89db78c02c2c4a5007247b7ecc177e45b0d6 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 28 Jul 2026 17:20:04 +0800 Subject: [PATCH 08/15] Regenerate migrations after rebase --- ... 20260728090710_EventLogItems.Designer.cs} | 53 ++++++++++++++++++- ...ems.cs => 20260728090710_EventLogItems.cs} | 0 ... 20260728090700_EventLogItems.Designer.cs} | 45 +++++++++++++++- ...ems.cs => 20260728090700_EventLogItems.cs} | 0 4 files changed, 96 insertions(+), 2 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260728075200_EventLogItems.Designer.cs => 20260728090710_EventLogItems.Designer.cs} (86%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260728075200_EventLogItems.cs => 20260728090710_EventLogItems.cs} (100%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260728075138_EventLogItems.Designer.cs => 20260728090700_EventLogItems.Designer.cs} (86%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260728075138_EventLogItems.cs => 20260728090700_EventLogItems.cs} (100%) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs similarity index 86% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs index 9fef432e27..1fd7d3c96b 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260728075200_EventLogItems")] + [Migration("20260728090710_EventLogItems")] partial class EventLogItems { /// @@ -26,6 +26,23 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => { b.Property("Id") @@ -117,6 +134,12 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("exception_type"); + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + b.Property("FirstTimeOfFailure") .HasColumnType("timestamp with time zone") .HasColumnName("first_time_of_failure"); @@ -206,6 +229,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("ConversationId") .HasDatabaseName("ix_failed_messages_conversation_id"); + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + b.HasIndex("QueueAddress") .HasDatabaseName("ix_failed_messages_queue_address"); @@ -305,6 +331,31 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("known_endpoints", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.TrialMetadataEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("TrialEndDate") + .HasColumnType("date") + .HasColumnName("trial_end_date"); + + b.HasKey("Id") + .HasName("pk_trial_metadata"); + + b.ToTable("trial_metadata", (string)null); + + b.HasData( + new + { + Id = 1 + }); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => { b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728075200_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs similarity index 86% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs index 2eb91c6990..3640073e32 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260728075138_EventLogItems")] + [Migration("20260728090700_EventLogItems")] partial class EventLogItems { /// @@ -25,6 +25,20 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => { b.Property("Id") @@ -98,6 +112,11 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("ExceptionType") .HasColumnType("nvarchar(max)"); + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + b.Property("FirstTimeOfFailure") .HasColumnType("datetime2"); @@ -166,6 +185,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("ConversationId"); + b.HasIndex("FailingEndpointAddress"); + b.HasIndex("QueueAddress"); b.HasIndex("ReceivingEndpointName"); @@ -245,6 +266,28 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("KnownEndpoints"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.TrialMetadataEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("TrialEndDate") + .HasColumnType("date"); + + b.HasKey("Id"); + + b.ToTable("TrialMetadata"); + + b.HasData( + new + { + Id = 1 + }); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => { b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728075138_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.cs From ab3177a3cd868521127fac96e348e47680d37cc7 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 29 Jul 2026 16:04:00 +0800 Subject: [PATCH 09/15] - Remove RavenDB specifics into the RavenDB persistence seam. - Seperate EventLogItem into a read/dto, and write model. The persistence now takes care of setting an Id, not the application layer - Add tests --- ... 20260729064621_EventLogItems.Designer.cs} | 24 ++++++----- ...ems.cs => 20260729064621_EventLogItems.cs} | 12 ++++-- ...SqlServiceControlDbContextModelSnapshot.cs | 22 +++++----- ... 20260729064551_EventLogItems.Designer.cs} | 21 ++++----- ...ems.cs => 20260729064551_EventLogItems.cs} | 12 ++++-- ...verServiceControlDbContextModelSnapshot.cs | 19 ++++---- .../Entities/EventLogItemEntity.cs | 7 +-- .../EventLogItemConfiguration.cs | 9 ++-- .../Implementation/EventLogDataStore.cs | 33 +++++++------- .../EventLogItemIdGenerator.cs | 17 ++++++++ .../EventLogDataStore.cs | 33 +++++++++++--- .../EventLogItemIdGeneratorTests.cs | 43 +++++++++++++++++++ .../Expiration/MessageExpiryTests.cs | 4 +- .../EFCore/EventLogDataStoreEFTests.cs | 11 +++-- .../EFCore/RetentionSweepTests.cs | 14 +++--- .../EventLogDataStoreTests.cs | 29 +++++++++---- .../EventLog/EventLogItem.cs | 5 ++- .../EventLog/EventLogItemView.cs | 22 ++++++++++ .../IEventLogDataStore.cs | 20 +++++---- .../EventLog/AuditEventLogWriter.cs | 4 +- .../EventLog/EventLogApiController.cs | 2 +- .../EventLog/EventLogMappingDefinition.cs | 2 - 22 files changed, 255 insertions(+), 110 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260728090710_EventLogItems.Designer.cs => 20260729064621_EventLogItems.Designer.cs} (96%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260728090710_EventLogItems.cs => 20260729064621_EventLogItems.cs} (81%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260728090700_EventLogItems.Designer.cs => 20260729064551_EventLogItems.Designer.cs} (95%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260728090700_EventLogItems.cs => 20260729064551_EventLogItems.cs} (81%) create mode 100644 src/ServiceControl.Persistence.RavenDB/DocumentIdGenerators/EventLogItemIdGenerator.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemIdGeneratorTests.cs create mode 100644 src/ServiceControl.Persistence/EventLog/EventLogItemView.cs diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs similarity index 96% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs index 1fd7d3c96b..4251be5ca6 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260728090710_EventLogItems")] + [Migration("20260729064621_EventLogItems")] partial class EventLogItems { /// @@ -54,8 +54,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Category") .IsRequired() - .HasMaxLength(255) - .HasColumnType("character varying(255)") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("category"); b.Property("Description") @@ -63,16 +63,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("description"); - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("character varying(600)") - .HasColumnName("event_log_item_id"); - b.Property("EventType") .IsRequired() - .HasMaxLength(255) - .HasColumnType("character varying(255)") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("event_type"); b.Property("RaisedAt") @@ -88,9 +82,17 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("severity"); + b.Property("UniqueEventId") + .HasColumnType("uuid") + .HasColumnName("unique_event_id"); + b.HasKey("Id") .HasName("pk_event_log_items"); + b.HasIndex("UniqueEventId") + .IsUnique() + .HasDatabaseName("ix_event_log_items_unique_event_id"); + b.HasIndex("RaisedAt", "Id") .IsDescending() .HasDatabaseName("ix_event_log_items_raised_at_id"); diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.cs similarity index 81% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.cs index 6238f09995..e6c3037c98 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260728090710_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.cs @@ -19,13 +19,13 @@ protected override void Up(MigrationBuilder migrationBuilder) { id = table.Column(type: "bigint", nullable: false) .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), - event_log_item_id = table.Column(type: "character varying(600)", maxLength: 600, nullable: false), + unique_event_id = table.Column(type: "uuid", nullable: false), description = table.Column(type: "text", nullable: false), severity = table.Column(type: "integer", nullable: false), raised_at = table.Column(type: "timestamp with time zone", nullable: false), related_to = table.Column>(type: "text[]", nullable: false), - category = table.Column(type: "character varying(255)", maxLength: 255, nullable: false), - event_type = table.Column(type: "character varying(255)", maxLength: 255, nullable: false) + category = table.Column(type: "character varying(450)", maxLength: 450, nullable: false), + event_type = table.Column(type: "character varying(450)", maxLength: 450, nullable: false) }, constraints: table => { @@ -37,6 +37,12 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "EventLogItems", columns: new[] { "raised_at", "id" }, descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "ix_event_log_items_unique_event_id", + table: "EventLogItems", + column: "unique_event_id", + unique: true); } /// diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index cdd95ce052..41b00e9307 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -51,8 +51,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Category") .IsRequired() - .HasMaxLength(255) - .HasColumnType("character varying(255)") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("category"); b.Property("Description") @@ -60,16 +60,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("description"); - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("character varying(600)") - .HasColumnName("event_log_item_id"); - b.Property("EventType") .IsRequired() - .HasMaxLength(255) - .HasColumnType("character varying(255)") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("event_type"); b.Property("RaisedAt") @@ -85,9 +79,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("severity"); + b.Property("UniqueEventId") + .HasColumnType("uuid") + .HasColumnName("unique_event_id"); + b.HasKey("Id") .HasName("pk_event_log_items"); + b.HasIndex("UniqueEventId") + .IsUnique() + .HasDatabaseName("ix_event_log_items_unique_event_id"); + b.HasIndex("RaisedAt", "Id") .IsDescending() .HasDatabaseName("ix_event_log_items_raised_at_id"); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs similarity index 95% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs index 3640073e32..50e71ffc52 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260728090700_EventLogItems")] + [Migration("20260729064551_EventLogItems")] partial class EventLogItems { /// @@ -49,22 +49,17 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Category") .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.Property("Description") .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("nvarchar(600)"); - b.Property("EventType") .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.Property("RaisedAt") .HasColumnType("datetime2"); @@ -76,8 +71,14 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Severity") .HasColumnType("int"); + b.Property("UniqueEventId") + .HasColumnType("uniqueidentifier"); + b.HasKey("Id"); + b.HasIndex("UniqueEventId") + .IsUnique(); + b.HasIndex("RaisedAt", "Id") .IsDescending(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.cs similarity index 81% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.cs index 345acc4054..ba6f815e1c 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260728090700_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.cs @@ -17,13 +17,13 @@ protected override void Up(MigrationBuilder migrationBuilder) { Id = table.Column(type: "bigint", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), - EventLogItemId = table.Column(type: "nvarchar(600)", maxLength: 600, nullable: false), + UniqueEventId = table.Column(type: "uniqueidentifier", nullable: false), Description = table.Column(type: "nvarchar(max)", nullable: false), Severity = table.Column(type: "int", nullable: false), RaisedAt = table.Column(type: "datetime2", nullable: false), RelatedTo = table.Column(type: "nvarchar(max)", nullable: false), - Category = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), - EventType = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false) + Category = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: false), + EventType = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: false) }, constraints: table => { @@ -35,6 +35,12 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "EventLogItems", columns: new[] { "RaisedAt", "Id" }, descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "IX_EventLogItems_UniqueEventId", + table: "EventLogItems", + column: "UniqueEventId", + unique: true); } /// diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index 7f5f923eed..339a4b0bbe 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -81,22 +81,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Category") .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.Property("Description") .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("EventLogItemId") - .IsRequired() - .HasMaxLength(600) - .HasColumnType("nvarchar(600)"); - b.Property("EventType") .IsRequired() - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.Property("RaisedAt") .HasColumnType("datetime2"); @@ -108,8 +103,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Severity") .HasColumnType("int"); + b.Property("UniqueEventId") + .HasColumnType("uniqueidentifier"); + b.HasKey("Id"); + b.HasIndex("UniqueEventId") + .IsUnique(); + b.HasIndex("RaisedAt", "Id") .IsDescending(); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs index d936c44f0d..ced3dee9f9 100644 --- a/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs +++ b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs @@ -4,11 +4,12 @@ namespace ServiceControl.Persistence.EFCore.Entities; public class EventLogItemEntity { + // The physical key only. Sequential so inserts stay at the tail of the index, and narrow so it + // is a cheap tiebreaker for RaisedAt paging. public long Id { get; set; } - // The API-visible identity, assigned by EventLogMappingDefinition as - // "EventLogItem/{Category}/{EventType}/{guid}" - public required string EventLogItemId { get; set; } + // A globally unique identity common between persisters (e.g. used in migrations). + public Guid UniqueEventId { get; set; } public required string Description { get; set; } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs index 8374f9a77b..6dbe6f19e7 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs @@ -11,16 +11,15 @@ public void Configure(EntityTypeBuilder builder) builder.ToTable("EventLogItems"); builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedOnAdd(); - // Long enough for "EventLogItem/{Category}/{EventType}/{guid}" form. Deliberately not - // indexed and not unique. Nothing looks an item up by it, and each id embeds a fresh Guid. - builder.Property(e => e.EventLogItemId).IsRequired().HasMaxLength(600); + builder.Property(e => e.UniqueEventId).IsRequired(); + builder.HasIndex(e => e.UniqueEventId).IsUnique(); builder.Property(e => e.Description).IsRequired(); builder.Property(e => e.Severity).IsRequired(); builder.Property(e => e.RaisedAt).IsRequired(); // RelatedTo is only ever returned to the API, never queried, so it does not warrant a child table. builder.Property(e => e.RelatedTo).IsRequired(); - builder.Property(e => e.Category).IsRequired().HasMaxLength(255); - builder.Property(e => e.EventType).IsRequired().HasMaxLength(255); + builder.Property(e => e.Category).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.EventType).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); // Every read is "order by RaisedAt descending" plus paging. The key is included as a // tiebreaker so that items sharing a RaisedAt do not shuffle between pages, and so that // MAX(RaisedAt) for the ETag is an index seek. diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index 7c2c2c2669..64dc5ef08f 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -8,12 +8,12 @@ namespace ServiceControl.Persistence.EFCore.Implementation; public class EventLogDataStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IEventLogDataStore { - public Task Add(EventLogItem logItem) => + public Task Add(EventLogItem logItem, Guid eventId) => ExecuteWithDbContext(async dbContext => { dbContext.EventLogItems.Add(new EventLogItemEntity { - EventLogItemId = logItem.Id, + UniqueEventId = eventId, Description = logItem.Description, Severity = logItem.Severity, RaisedAt = logItem.RaisedAt, @@ -25,9 +25,9 @@ public Task Add(EventLogItem logItem) => await dbContext.SaveChangesAsync(); }); - public Task<(IList? items, long total, string version)> GetEventLogItems( + public Task<(IList? items, long total, string version)> GetEventLogItems( PagingInfo pagingInfo, string? knownVersion = null) => - ExecuteWithDbContext<(IList? items, long total, string version)>(async dbContext => + ExecuteWithDbContext<(IList? items, long total, string version)>(async dbContext => { var query = dbContext.EventLogItems.AsNoTracking(); @@ -49,25 +49,28 @@ public Task Add(EventLogItem logItem) => return (null, total, version); } - var items = await query + var rows = await query // The key breaks ties so that items sharing a RaisedAt cannot shuffle between // pages. IX_EventLogItems_RaisedAt_Id is declared in exactly this order. .OrderByDescending(e => e.RaisedAt) .ThenByDescending(e => e.Id) .Skip(pagingInfo.Offset) .Take(pagingInfo.PageSize) - .Select(e => new EventLogItem - { - Id = e.EventLogItemId, - Description = e.Description, - Severity = e.Severity, - RaisedAt = e.RaisedAt, - RelatedTo = e.RelatedTo, - Category = e.Category, - EventType = e.EventType - }) .ToListAsync(); + // The id is stringified here, not in the query: SQL Server converts uniqueidentifier + // to uppercase hex, while Guid.ToString() and PostgreSQL both produce lowercase. + var items = rows.Select(e => new EventLogItemView + { + Id = e.UniqueEventId.ToString(), + Description = e.Description, + Severity = e.Severity, + RaisedAt = e.RaisedAt, + RelatedTo = e.RelatedTo, + Category = e.Category, + EventType = e.EventType + }).ToList(); + return (items, total, version); }); diff --git a/src/ServiceControl.Persistence.RavenDB/DocumentIdGenerators/EventLogItemIdGenerator.cs b/src/ServiceControl.Persistence.RavenDB/DocumentIdGenerators/EventLogItemIdGenerator.cs new file mode 100644 index 0000000000..03ee2bf93e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DocumentIdGenerators/EventLogItemIdGenerator.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.RavenDB +{ + using System; + + static class EventLogItemIdGenerator + { + public const string DocumentIdPrefix = "EventLogItem"; + + public static string MakeDocumentId(string category, string eventType, Guid eventId) => + $"{DocumentIdPrefix}/{category}/{eventType}/{eventId}"; + + // The final segment is safe: a category is a namespace segment and an event type a type + // name, so neither can contain a separator. + public static Guid GetEventIdFromDocumentId(string documentId) => + Guid.Parse(documentId[(documentId.LastIndexOf('/') + 1)..]); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs index b6d7abbbcb..6f6d019739 100644 --- a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.RavenDB { + using System; using System.Collections.Generic; using System.Threading.Tasks; using EventLog; @@ -8,10 +9,12 @@ class EventLogDataStore(IRavenSessionProvider sessionProvider, ExpirationManager expirationManager) : IEventLogDataStore { - public async Task Add(EventLogItem logItem) + public async Task Add(EventLogItem logItem, Guid eventId) { using var session = await sessionProvider.OpenSession(); - await session.StoreAsync(logItem); + await session.StoreAsync( + logItem, + EventLogItemIdGenerator.MakeDocumentId(logItem.Category, logItem.EventType, eventId)); // Retention on RavenDB is per-document expiry metadata stamped at write time, not a // sweep. It has to be set here, on the only write path, or items never expire. @@ -20,11 +23,11 @@ public async Task Add(EventLogItem logItem) await session.SaveChangesAsync(); } - public async Task<(IList, long, string)> GetEventLogItems( + public async Task<(IList, long, string)> GetEventLogItems( PagingInfo pagingInfo, string knownVersion = null) { using var session = await sessionProvider.OpenSession(); - var results = await session + var documents = await session .Query() .Statistics(out var stats) .OrderByDescending(p => p.RaisedAt) @@ -33,10 +36,28 @@ public async Task Add(EventLogItem logItem) var version = stats.ResultEtag.ToString(); - // For robustness and consistency. Decide 304s at the controller level. + // For robustness and consistency. Decide 304s at the controller level. var unchanged = knownVersion is not null && knownVersion == version; - return (unchanged ? null : results, stats.TotalResults, version); + if (unchanged) + { + return (null, stats.TotalResults, version); + } + + // The id lives in document metadata rather than on the document, so it has to be read + // from the session while it is still open. + var items = documents.ConvertAll(document => new EventLogItemView + { + Id = session.Advanced.GetDocumentId(document), + Description = document.Description, + Severity = document.Severity, + RaisedAt = document.RaisedAt, + RelatedTo = document.RelatedTo, + Category = document.Category, + EventType = document.EventType + }); + + return (items, stats.TotalResults, version); } } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemIdGeneratorTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemIdGeneratorTests.cs new file mode 100644 index 0000000000..561c56d8a6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemIdGeneratorTests.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DocumentIdGenerators +{ + using System; + using NUnit.Framework; + using ServiceControl.Persistence.RavenDB; + + [TestFixture] + class EventLogItemIdGeneratorTests + { + // Documents already in a customer's store carry ids of this shape, and RavenDB reads a document's id from metadata rather than + // from a field, so a changed format orphans everything written before the upgrade. + [Test] + public void Document_id_keeps_this_shape() + { + var eventId = Guid.Parse("2f6d1b4e-8c3a-4d5f-9a1b-7e0c2d3f4a5b"); + + var documentId = EventLogItemIdGenerator.MakeDocumentId("CustomChecks", "CustomCheckFailed", eventId); + + Assert.That(documentId, Is.EqualTo("EventLogItem/CustomChecks/CustomCheckFailed/2f6d1b4e-8c3a-4d5f-9a1b-7e0c2d3f4a5b")); + } + + [Test] + public void Event_id_is_recovered_from_a_legacy_document_id() + { + var legacyDocumentId = "EventLogItem/Recoverability/MessageFailed/a1b2c3d4-e5f6-4788-9a0b-1c2d3e4f5a6b"; + + var eventId = EventLogItemIdGenerator.GetEventIdFromDocumentId(legacyDocumentId); + + Assert.That(eventId, Is.EqualTo(Guid.Parse("a1b2c3d4-e5f6-4788-9a0b-1c2d3e4f5a6b"))); + } + + [Test] + public void Composing_then_recovering_returns_the_same_event_id() + { + var eventId = Guid.CreateVersion7(); + + var recovered = EventLogItemIdGenerator.GetEventIdFromDocumentId( + EventLogItemIdGenerator.MakeDocumentId("Monitoring", "EndpointStarted", eventId)); + + Assert.That(recovered, Is.EqualTo(eventId)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs index b28475e17f..665cab6ae8 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs @@ -246,7 +246,9 @@ public async Task EventLogItemShouldExpire() { await DisableExpiration(); - await EventLogDataStore.Add(new EventLogItem()); + await EventLogDataStore.Add( + new EventLogItem { Category = "Recoverability", EventType = "MessageFailed" }, + Guid.CreateVersion7()); await CompleteDatabaseOperation(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs index 4c7539270d..5016883243 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -24,9 +24,9 @@ public async Task Items_sharing_a_raised_at_page_without_overlap_or_omission() for (var i = 0; i < 10; i++) { - var item = CreateLogItem($"Collision{i}", sameInstant); - expected.Add(item.Id); - await EventLogDataStore.Add(item); + var eventId = Guid.CreateVersion7(); + expected.Add(eventId.ToString()); + await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant), eventId); } await CompleteDatabaseOperation(); @@ -53,7 +53,7 @@ public async Task Paging_order_is_repeatable_when_raised_at_collides() for (var i = 0; i < 6; i++) { - await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant)); + await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant), Guid.CreateVersion7()); } await CompleteDatabaseOperation(); @@ -73,7 +73,7 @@ public async Task Version_changes_when_rows_are_deleted_behind_the_interface() for (var i = 0; i < 3; i++) { - await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); } await CompleteDatabaseOperation(); @@ -110,7 +110,6 @@ async Task DeleteOldest() static EventLogItem CreateLogItem(string eventType, DateTime raisedAt) => new() { - Id = $"EventLogItem/Recoverability/{eventType}/{Guid.NewGuid()}", Category = "Recoverability", EventType = eventType, Description = $"{eventType} occurred", diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index e14a31c46d..2bf39a28bc 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -155,7 +155,7 @@ public async Task Deletes_event_log_items_past_the_events_cutoff() await RunRetentionSweep(); - var remaining = await GetEventLogItemIds(); + var remaining = await GetRemainingMarkers(); using (Assert.EnterMultipleScope()) { @@ -175,7 +175,7 @@ public async Task Event_log_retention_is_independent() await RunRetentionSweep(); - Assert.That(await GetEventLogItemIds(), Does.Not.Contain("old-event")); + Assert.That(await GetRemainingMarkers(), Does.Not.Contain("old-event")); } [Test] @@ -200,10 +200,10 @@ public async Task Sweeping_event_log_items_changes_the_version() } } - static EventLogItemEntity EventLogRow(string id, DateTime raisedAt) => new() + static EventLogItemEntity EventLogRow(string marker, DateTime raisedAt) => new() { - EventLogItemId = id, - Description = "swept", + UniqueEventId = Guid.CreateVersion7(), + Description = marker, Severity = Severity.Info, RaisedAt = raisedAt, RelatedTo = [], @@ -211,9 +211,9 @@ public async Task Sweeping_event_log_items_changes_the_version() EventType = "MessageFailed" }; - async Task> GetEventLogItemIds() + async Task> GetRemainingMarkers() { var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 100)); - return [.. items.Select(i => i.Id)]; + return [.. items.Select(i => i.Description)]; } } diff --git a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs index 4d04c4c1c8..9ee3fdd080 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs @@ -18,7 +18,7 @@ public async Task Added_item_is_returned_with_all_values_intact() logItem.Description = "Message processing failed"; logItem.RelatedTo = ["/message/abc123", "/endpoint/Sales"]; - await EventLogDataStore.Add(logItem); + await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); await CompleteDatabaseOperation(); var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); @@ -27,7 +27,6 @@ public async Task Added_item_is_returned_with_all_values_intact() var stored = items[0]; using (Assert.EnterMultipleScope()) { - Assert.That(stored.Id, Is.EqualTo(logItem.Id)); Assert.That(stored.Description, Is.EqualTo("Message processing failed")); Assert.That(stored.Severity, Is.EqualTo(Severity.Error)); Assert.That(stored.RaisedAt, Is.EqualTo(raisedAt)); @@ -37,13 +36,28 @@ public async Task Added_item_is_returned_with_all_values_intact() } } + [Test] + public async Task The_supplied_event_id_is_recoverable_from_the_stored_item() + { + var eventId = Guid.CreateVersion7(); + var logItem = CreateLogItem("MessageFailed", new DateTime(2026, 7, 22, 10, 30, 0, DateTimeKind.Utc)); + + await EventLogDataStore.Add(logItem, eventId); + await CompleteDatabaseOperation(); + + var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + // Not equality: Different persisters may return the event id in different formats. + Assert.That(items[0].Id, Does.Contain(eventId.ToString())); + } + [Test] public async Task Item_with_no_related_links_round_trips_as_empty() { var logItem = CreateLogItem("EndpointStarted", DateTime.UtcNow); logItem.RelatedTo = []; - await EventLogDataStore.Add(logItem); + await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); await CompleteDatabaseOperation(); var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); @@ -55,9 +69,9 @@ public async Task Item_with_no_related_links_round_trips_as_empty() public async Task Items_are_returned_most_recently_raised_first() { var baseTime = new DateTime(2026, 7, 22, 9, 0, 0, DateTimeKind.Utc); - await EventLogDataStore.Add(CreateLogItem("Oldest", baseTime)); - await EventLogDataStore.Add(CreateLogItem("Newest", baseTime.AddMinutes(2))); - await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1))); + await EventLogDataStore.Add(CreateLogItem("Oldest", baseTime), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem("Newest", baseTime.AddMinutes(2)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1)), Guid.CreateVersion7()); await CompleteDatabaseOperation(); var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); @@ -220,7 +234,7 @@ async Task AddItems(int count) for (var i = 0; i < count; i++) { - await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); } await CompleteDatabaseOperation(); @@ -228,7 +242,6 @@ async Task AddItems(int count) static EventLogItem CreateLogItem(string eventType, DateTime raisedAt) => new() { - Id = $"EventLogItem/Recoverability/{eventType}/{Guid.NewGuid()}", Category = "Recoverability", EventType = eventType, Description = $"{eventType} occurred", diff --git a/src/ServiceControl.Persistence/EventLog/EventLogItem.cs b/src/ServiceControl.Persistence/EventLog/EventLogItem.cs index c726ccab14..563416ecd7 100644 --- a/src/ServiceControl.Persistence/EventLog/EventLogItem.cs +++ b/src/ServiceControl.Persistence/EventLog/EventLogItem.cs @@ -3,9 +3,12 @@ using System; using System.Collections.Generic; + /// + /// One event log item as written. Carries no identity: each persister assigns that itself and + /// surfaces it on when the item is read back. + /// public class EventLogItem { - public string Id { get; set; } public string Description { get; set; } public Severity Severity { get; set; } public DateTime RaisedAt { get; set; } diff --git a/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs b/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs new file mode 100644 index 0000000000..5f6bf787f0 --- /dev/null +++ b/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.EventLog +{ + using System; + using System.Collections.Generic; + + /// + /// One event log item as read back and returned by the API. + /// + public class EventLogItemView + { + /// + /// Assigned by whichever persister stored the item + /// + public string Id { get; set; } + public string Description { get; set; } + public Severity Severity { get; set; } + public DateTime RaisedAt { get; set; } + public List RelatedTo { get; set; } + public string Category { get; set; } + public string EventType { get; set; } + } +} diff --git a/src/ServiceControl.Persistence/IEventLogDataStore.cs b/src/ServiceControl.Persistence/IEventLogDataStore.cs index 3e6a329a07..cc92d3bc96 100644 --- a/src/ServiceControl.Persistence/IEventLogDataStore.cs +++ b/src/ServiceControl.Persistence/IEventLogDataStore.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence { + using System; using System.Collections.Generic; using System.Threading.Tasks; using EventLog; @@ -16,11 +17,14 @@ public interface IEventLogDataStore { /// - /// Persists a single event log item. arrives fully formed from - /// EventLogMappingDefinition.Apply, including its Id, which a persister must - /// treat as opaque. + /// Persists a single event log item. /// - Task Add(EventLogItem logItem); + /// The item to store. + /// + /// The item's portable/global identity, minted by the caller, and survives a + /// move between persisters. + /// + Task Add(EventLogItem logItem, Guid eventId); /// /// Returns one page of event log items, newest RaisedAt first. @@ -30,13 +34,13 @@ public interface IEventLogDataStore /// The version the caller already holds, or null if it holds none. /// /// - /// items — the requested page, which may be empty; - /// total — the number of items in the store, independent of the page size; - /// version — an opaque cache validator surfaced as the ETag response header by + /// items: the requested page, which may be empty; + /// total: the number of items in the store, independent of the page size; + /// version: an opaque cache validator surfaced as the ETag response header by /// EventLogApiController. It must change when retention removes items, not only when /// one is added, since nothing else tells a client its cached page is now wrong. /// - Task<(IList items, long total, string version)> GetEventLogItems( + Task<(IList items, long total, string version)> GetEventLogItems( PagingInfo pagingInfo, string knownVersion = null); } } \ No newline at end of file diff --git a/src/ServiceControl/EventLog/AuditEventLogWriter.cs b/src/ServiceControl/EventLog/AuditEventLogWriter.cs index aeb14eafc9..69e15f78f4 100644 --- a/src/ServiceControl/EventLog/AuditEventLogWriter.cs +++ b/src/ServiceControl/EventLog/AuditEventLogWriter.cs @@ -1,5 +1,6 @@ namespace ServiceControl.EventLog { + using System; using System.Threading; using System.Threading.Tasks; using Infrastructure.DomainEvents; @@ -25,8 +26,9 @@ public async Task Handle(IDomainEvent message, CancellationToken cancellationTok } var logItem = mappings.ApplyMapping(message); + var eventId = Guid.CreateVersion7(); - await dataStore.Add(logItem); + await dataStore.Add(logItem, eventId); } readonly IEventLogDataStore dataStore; diff --git a/src/ServiceControl/EventLog/EventLogApiController.cs b/src/ServiceControl/EventLog/EventLogApiController.cs index 253e2275fd..765b72fd0d 100644 --- a/src/ServiceControl/EventLog/EventLogApiController.cs +++ b/src/ServiceControl/EventLog/EventLogApiController.cs @@ -18,7 +18,7 @@ public class EventLogApiController(IEventLogDataStore logDataStore) : Controller [Authorize(Policy = Permissions.ErrorEventLogView)] [Route("eventlogitems")] [HttpGet] - public async Task>> Items([FromQuery] PagingInfo pagingInfo) + public async Task>> Items([FromQuery] PagingInfo pagingInfo) { // The Trim handles both ETag formats (quoted and unquoted) deliberately. diff --git a/src/ServiceControl/EventLog/EventLogMappingDefinition.cs b/src/ServiceControl/EventLog/EventLogMappingDefinition.cs index 85c9038b58..c532b17289 100644 --- a/src/ServiceControl/EventLog/EventLogMappingDefinition.cs +++ b/src/ServiceControl/EventLog/EventLogMappingDefinition.cs @@ -12,10 +12,8 @@ abstract class EventLogMappingDefinition : IEventLogMappingDefinition where T public EventLogItem Apply(IDomainEvent @event) { var eventMessage = (T)@event; - var eventId = Guid.NewGuid().ToString(); var item = new EventLogItem { - Id = $"EventLogItem/{Category}/{typeof(T).Name}/{eventId}", Category = Category, RaisedAt = raisedAtFunc(eventMessage), Description = descriptionFunc(eventMessage), From 3bef62a74e74b93696e4c4595c3717e950678e5a Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 29 Jul 2026 16:33:27 +0800 Subject: [PATCH 10/15] Update EventLogItems migration for PostgreSQL and SQL Server --- ... 20260729082915_EventLogItems.Designer.cs} | 46 +++++++++++- ...ems.cs => 20260729082915_EventLogItems.cs} | 0 ... 20260729082911_EventLogItems.Designer.cs} | 37 +++++++++- ...ems.cs => 20260729082911_EventLogItems.cs} | 0 ...verServiceControlDbContextModelSnapshot.cs | 70 +++++++++---------- 5 files changed, 116 insertions(+), 37 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260729064621_EventLogItems.Designer.cs => 20260729082915_EventLogItems.Designer.cs} (89%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260729064621_EventLogItems.cs => 20260729082915_EventLogItems.cs} (100%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260729064551_EventLogItems.Designer.cs => 20260729082911_EventLogItems.Designer.cs} (89%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260729064551_EventLogItems.cs => 20260729082911_EventLogItems.cs} (100%) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs similarity index 89% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs index 4251be5ca6..fecd33ee3b 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260729064621_EventLogItems")] + [Migration("20260729082915_EventLogItems")] partial class EventLogItems { /// @@ -100,6 +100,50 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("EventLogItems", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { b.Property("UniqueMessageId") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729064621_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs similarity index 89% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs index 50e71ffc52..cc49c4f5cc 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260729064551_EventLogItems")] + [Migration("20260729082911_EventLogItems")] partial class EventLogItems { /// @@ -85,6 +85,41 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("EventLogItems", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { b.Property("UniqueMessageId") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729064551_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index 339a4b0bbe..59473326f4 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -36,41 +36,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("EndpointSettings"); }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => - { - b.Property("UniqueMessageId") - .HasColumnType("uniqueidentifier"); - - b.Property("Body") - .IsRequired() - .HasColumnType("varbinary(max)"); - - b.Property("BodyStoredExternally") - .HasColumnType("bit"); - - b.Property("ExceptionInfo") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("FailedAt") - .HasColumnType("datetime2"); - - b.Property("HeadersJson") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("MessageId") - .IsRequired() - .HasMaxLength(450) - .HasColumnType("nvarchar(450)"); - - b.HasKey("UniqueMessageId"); - - b.HasIndex("FailedAt"); - - b.ToTable("FailedErrorImports"); - }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => { b.Property("Id") @@ -117,6 +82,41 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("EventLogItems", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { b.Property("UniqueMessageId") From 42011db8f9f506bd3265350b9747d5f8d3848596 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 29 Jul 2026 20:29:02 +0800 Subject: [PATCH 11/15] Refactor GetEventLogItems method to return QueryResult type and update related tests --- .../Implementation/EventLogDataStore.cs | 9 +- .../EventLogDataStore.cs | 18 ++-- .../Expiration/MessageExpiryTests.cs | 4 +- .../EFCore/EventLogDataStoreEFTests.cs | 18 ++-- .../EFCore/RetentionSweepTests.cs | 10 +- .../EventLogDataStoreTests.cs | 101 +++++++++++------- .../EventLogWriterTests.cs | 12 +-- .../IEventLogDataStore.cs | 15 +-- .../Infrastructure/QueryResult.cs | 9 ++ .../EventLog/EventLogApiController.cs | 11 +- 10 files changed, 120 insertions(+), 87 deletions(-) diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index 64dc5ef08f..170836af8b 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -25,9 +25,9 @@ public Task Add(EventLogItem logItem, Guid eventId) => await dbContext.SaveChangesAsync(); }); - public Task<(IList? items, long total, string version)> GetEventLogItems( + public Task>> GetEventLogItems( PagingInfo pagingInfo, string? knownVersion = null) => - ExecuteWithDbContext<(IList? items, long total, string version)>(async dbContext => + ExecuteWithDbContext(async dbContext => { var query = dbContext.EventLogItems.AsNoTracking(); @@ -40,13 +40,14 @@ public Task Add(EventLogItem logItem, Guid eventId) => var total = stats?.Total ?? 0; var version = Version(total, stats?.Newest); + var queryStats = new QueryStatsInfo(version, total, isStale: false); // The point of knownVersion. Everything above is index work. // If the caller already has the latest version, skip the rest of the query. // No database round trip is needed. No response body is needed. if (knownVersion is not null && knownVersion == version) { - return (null, total, version); + return QueryResult>.Unchanged(queryStats); } var rows = await query @@ -71,7 +72,7 @@ public Task Add(EventLogItem logItem, Guid eventId) => EventType = e.EventType }).ToList(); - return (items, total, version); + return new QueryResult>(items, queryStats); }); // Synthesised version ID to be used for an ETag, using total count and the newest item's RaisedAt timestamp. diff --git a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs index 6f6d019739..a1ef4c5bf1 100644 --- a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs @@ -23,7 +23,7 @@ await session.StoreAsync( await session.SaveChangesAsync(); } - public async Task<(IList, long, string)> GetEventLogItems( + public async Task>> GetEventLogItems( PagingInfo pagingInfo, string knownVersion = null) { using var session = await sessionProvider.OpenSession(); @@ -34,18 +34,16 @@ await session.StoreAsync( .Paging(pagingInfo) .ToListAsync(); - var version = stats.ResultEtag.ToString(); + var queryStats = stats.ToQueryStatsInfo(); - // For robustness and consistency. Decide 304s at the controller level. - var unchanged = knownVersion is not null && knownVersion == version; - - if (unchanged) + // The validator comes off the query statistics, so the page cannot be + // skipped. Only the projection below is saved. + if (knownVersion is not null && knownVersion == queryStats.ETag) { - return (null, stats.TotalResults, version); + return QueryResult>.Unchanged(queryStats); } - // The id lives in document metadata rather than on the document, so it has to be read - // from the session while it is still open. + // The id lives in document metadata rather than on the document var items = documents.ConvertAll(document => new EventLogItemView { Id = session.Advanced.GetDocumentId(document), @@ -57,7 +55,7 @@ await session.StoreAsync( EventType = document.EventType }); - return (items, stats.TotalResults, version); + return new QueryResult>(items, queryStats); } } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs index 665cab6ae8..07daacb443 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs @@ -252,7 +252,7 @@ await EventLogDataStore.Add( await CompleteDatabaseOperation(); - var (logItems, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(1, 1)); + var logItems = (await EventLogDataStore.GetEventLogItems(new PagingInfo(1, 1))).Results; Assert.That(logItems, Has.Count.EqualTo(1), "Event log items should be available to query."); @@ -260,7 +260,7 @@ await EventLogDataStore.Add( await WaitUntil(async () => { - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(1, 1)); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo(1, 1))).Results; return items.Count == 0; }, "Event log items should be removed after expiration period elapses."); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs index 5016883243..b9c93a7b1a 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -34,7 +34,7 @@ public async Task Items_sharing_a_raised_at_page_without_overlap_or_omission() var paged = new List(); for (var page = 1; page <= 5; page++) { - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: page, pageSize: 2)); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: page, pageSize: 2))).Results; paged.AddRange(items.Select(i => i.Id)); } @@ -58,8 +58,8 @@ public async Task Paging_order_is_repeatable_when_raised_at_collides() await CompleteDatabaseOperation(); - var (first, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6)); - var (second, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6)); + var first = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6))).Results; + var second = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 6))).Results; Assert.That(second.Select(i => i.Id), Is.EqualTo(first.Select(i => i.Id)).AsCollection); } @@ -78,19 +78,19 @@ public async Task Version_changes_when_rows_are_deleted_behind_the_interface() await CompleteDatabaseOperation(); - var (_, totalBefore, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); - Assert.That(totalBefore, Is.EqualTo(3)); + var before = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + Assert.That(before.QueryStats.TotalCount, Is.EqualTo(3)); - // Delete the OLDEST row — the one a retention sweep would take. A rowversion or MAX(RaisedAt) + // Delete the OLDEST row, the one a retention sweep would take. A rowversion or MAX(RaisedAt) // scheme would be entirely blind to this; only the count term catches it. await DeleteOldest(); - var (_, totalAfter, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var after = await EventLogDataStore.GetEventLogItems(new PagingInfo()); using (Assert.EnterMultipleScope()) { - Assert.That(totalAfter, Is.EqualTo(2)); - Assert.That(versionAfter, Is.Not.EqualTo(versionBefore), "a delete must invalidate the client's cached page"); + Assert.That(after.QueryStats.TotalCount, Is.EqualTo(2)); + Assert.That(after.QueryStats.ETag, Is.Not.EqualTo(before.QueryStats.ETag), "a delete must invalidate the client's cached page"); } } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 2bf39a28bc..53a4104bf3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -186,17 +186,17 @@ public async Task Sweeping_event_log_items_changes_the_version() await Store(EventLogRow("expired", Now.AddDays(-15))); await Store(EventLogRow("fresh", Now.AddDays(-1))); - var (_, _, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var versionBefore = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).QueryStats.ETag; await RunRetentionSweep(); - var (_, total, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var after = await EventLogDataStore.GetEventLogItems(new PagingInfo()); using (Assert.EnterMultipleScope()) { - Assert.That(total, Is.EqualTo(1)); + Assert.That(after.QueryStats.TotalCount, Is.EqualTo(1)); // The count term of the version exists precisely so that retention invalidates client caches. - Assert.That(versionAfter, Is.Not.EqualTo(versionBefore)); + Assert.That(after.QueryStats.ETag, Is.Not.EqualTo(versionBefore)); } } @@ -213,7 +213,7 @@ public async Task Sweeping_event_log_items_changes_the_version() async Task> GetRemainingMarkers() { - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 100)); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 100))).Results; return [.. items.Select(i => i.Description)]; } } diff --git a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs index 9ee3fdd080..bc93896573 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs @@ -21,7 +21,7 @@ public async Task Added_item_is_returned_with_all_values_intact() await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); await CompleteDatabaseOperation(); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; Assert.That(items, Has.Count.EqualTo(1)); var stored = items[0]; @@ -45,7 +45,7 @@ public async Task The_supplied_event_id_is_recoverable_from_the_stored_item() await EventLogDataStore.Add(logItem, eventId); await CompleteDatabaseOperation(); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; // Not equality: Different persisters may return the event id in different formats. Assert.That(items[0].Id, Does.Contain(eventId.ToString())); @@ -60,7 +60,7 @@ public async Task Item_with_no_related_links_round_trips_as_empty() await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); await CompleteDatabaseOperation(); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; Assert.That(items[0].RelatedTo, Is.Empty); } @@ -74,20 +74,32 @@ public async Task Items_are_returned_most_recently_raised_first() await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1)), Guid.CreateVersion7()); await CompleteDatabaseOperation(); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; - Assert.That(items.Select(i => i.EventType), Is.EqualTo(new[] { "Newest", "Middle", "Oldest" })); + Assert.That(items.Select(i => i.EventType), Is.EqualTo(["Newest", "Middle", "Oldest"])); } [Test] public async Task Empty_store_returns_no_items() { - var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo()); using (Assert.EnterMultipleScope()) { - Assert.That(items, Is.Empty); - Assert.That(total, Is.Zero); + Assert.That(result.Results, Is.Empty); + Assert.That(result.QueryStats.TotalCount, Is.Zero); + } + } + + [Test] + public async Task Empty_store_is_a_page_of_nothing_rather_than_not_modified() + { + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.NotModified, Is.False, "an empty store still has a representation to return"); + Assert.That(result.Results, Is.Not.Null); } } @@ -96,12 +108,12 @@ public async Task Page_size_limits_returned_items_but_not_the_total() { await AddItems(5); - var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2)); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2)); using (Assert.EnterMultipleScope()) { - Assert.That(items, Has.Count.EqualTo(2)); - Assert.That(total, Is.EqualTo(5)); + Assert.That(result.Results, Has.Count.EqualTo(2)); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(5)); } } @@ -110,8 +122,8 @@ public async Task Later_pages_continue_where_the_previous_page_ended() { await AddItems(5); - var (firstPage, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2)); - var (secondPage, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 2, pageSize: 2)); + var firstPage = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 1, pageSize: 2))).Results; + var secondPage = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 2, pageSize: 2))).Results; using (Assert.EnterMultipleScope()) { @@ -125,12 +137,12 @@ public async Task Final_page_returns_only_the_remaining_items() { await AddItems(5); - var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 3, pageSize: 2)); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 3, pageSize: 2)); using (Assert.EnterMultipleScope()) { - Assert.That(items, Has.Count.EqualTo(1)); - Assert.That(total, Is.EqualTo(5)); + Assert.That(result.Results, Has.Count.EqualTo(1)); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(5)); } } @@ -139,12 +151,12 @@ public async Task Page_beyond_the_end_returns_no_items() { await AddItems(2); - var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 5, pageSize: 2)); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(page: 5, pageSize: 2)); using (Assert.EnterMultipleScope()) { - Assert.That(items, Is.Empty); - Assert.That(total, Is.EqualTo(2)); + Assert.That(result.Results, Is.Empty); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(2)); } } @@ -152,10 +164,10 @@ public async Task Page_beyond_the_end_returns_no_items() public async Task Version_changes_when_an_item_is_added() { await AddItems(1); - var (_, _, versionBefore) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var versionBefore = await CurrentVersion(); await AddItems(1); - var (_, _, versionAfter) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var versionAfter = await CurrentVersion(); Assert.That(versionAfter, Is.Not.EqualTo(versionBefore)); } @@ -165,37 +177,41 @@ public async Task Version_is_stable_while_nothing_changes() { await AddItems(2); - var (_, _, firstRead) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); - var (_, _, secondRead) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var firstRead = await CurrentVersion(); + var secondRead = await CurrentVersion(); Assert.That(secondRead, Is.EqualTo(firstRead)); } [Test] - public async Task Matching_known_version_reports_no_items() + public async Task Matching_known_version_reports_not_modified() { await AddItems(3); - var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var version = await CurrentVersion(); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); - Assert.That(items, Is.Null, "a caller already holding the current version must be told so, not handed the page again"); + using (Assert.EnterMultipleScope()) + { + Assert.That(result.NotModified, Is.True, "a caller already holding the current version must be told so, not handed the page again"); + Assert.That(result.Results, Is.Null, "a not-modified result carries no page"); + } } [Test] public async Task Matching_known_version_still_reports_total_and_version() { await AddItems(3); - var (_, _, version) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var version = await CurrentVersion(); - var (_, total, versionAgain) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(), version); using (Assert.EnterMultipleScope()) { // The controller sets Total-Count and ETag on the 304, so neither may be dropped // just because the page was not fetched. - Assert.That(total, Is.EqualTo(3)); - Assert.That(versionAgain, Is.EqualTo(version)); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(3)); + Assert.That(result.QueryStats.ETag, Is.EqualTo(version)); } } @@ -203,18 +219,18 @@ public async Task Matching_known_version_still_reports_total_and_version() public async Task Stale_known_version_returns_the_page() { await AddItems(2); - var (_, _, staleVersion) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var staleVersion = await CurrentVersion(); await AddItems(1); - var (items, total, freshVersion) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), staleVersion); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(), staleVersion); using (Assert.EnterMultipleScope()) { - Assert.That(items, Is.Not.Null); - Assert.That(items, Has.Count.EqualTo(3)); - Assert.That(total, Is.EqualTo(3)); - Assert.That(freshVersion, Is.Not.EqualTo(staleVersion)); + Assert.That(result.NotModified, Is.False); + Assert.That(result.Results, Has.Count.EqualTo(3)); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(3)); + Assert.That(result.QueryStats.ETag, Is.Not.EqualTo(staleVersion)); } } @@ -223,11 +239,18 @@ public async Task Unrecognised_known_version_returns_the_page() { await AddItems(2); - var (items, _, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo(), "not-a-version-this-store-ever-issued"); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo(), "not-a-version-this-store-ever-issued"); - Assert.That(items, Is.Not.Null, "an unrecognised validator must be treated as a cache miss, never as a match"); + using (Assert.EnterMultipleScope()) + { + Assert.That(result.NotModified, Is.False, "an unrecognised validator must be treated as a cache miss, never as a match"); + Assert.That(result.Results, Is.Not.Null); + } } + async Task CurrentVersion() => + (await EventLogDataStore.GetEventLogItems(new PagingInfo())).QueryStats.ETag; + async Task AddItems(int count) { var baseTime = new DateTime(2026, 7, 22, 8, 0, 0, DateTimeKind.Utc); diff --git a/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs index c22c76c93d..68c36d2173 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogWriterTests.cs @@ -18,13 +18,13 @@ public async Task A_mapped_domain_event_is_persisted_and_can_be_read_back() await writer.Handle(new SomethingHappened { What = "it happened" }, CancellationToken.None); await CompleteDatabaseOperation(); - var (items, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo()); using (Assert.EnterMultipleScope()) { - Assert.That(total, Is.EqualTo(1)); - Assert.That(items.Single().Description, Is.EqualTo("it happened")); - Assert.That(items.Single().EventType, Is.EqualTo(nameof(SomethingHappened))); + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(1)); + Assert.That(result.Results.Single().Description, Is.EqualTo("it happened")); + Assert.That(result.Results.Single().EventType, Is.EqualTo(nameof(SomethingHappened))); } } @@ -36,9 +36,9 @@ public async Task An_unmapped_domain_event_is_ignored() await writer.Handle(new NothingMapsThis(), CancellationToken.None); await CompleteDatabaseOperation(); - var (_, total, _) = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo()); - Assert.That(total, Is.Zero, "only events with a mapping under EventLog\\Definitions are recorded"); + Assert.That(result.QueryStats.TotalCount, Is.Zero, "only events with a mapping under EventLog\\Definitions are recorded"); } AuditEventLogWriter CreateWriter() => diff --git a/src/ServiceControl.Persistence/IEventLogDataStore.cs b/src/ServiceControl.Persistence/IEventLogDataStore.cs index cc92d3bc96..34ff1a9d48 100644 --- a/src/ServiceControl.Persistence/IEventLogDataStore.cs +++ b/src/ServiceControl.Persistence/IEventLogDataStore.cs @@ -31,16 +31,19 @@ public interface IEventLogDataStore /// /// Which page to return. /// - /// The version the caller already holds, or null if it holds none. + /// The version the caller already holds, or null if it holds none. When it matches, the + /// result is and carries no page. /// /// - /// items: the requested page, which may be empty; - /// total: the number of items in the store, independent of the page size; - /// version: an opaque cache validator surfaced as the ETag response header by - /// EventLogApiController. It must change when retention removes items, not only when + /// : the requested page, which may be empty. + /// : the number of items in the store, independent of the + /// page size, and populated even when nothing was modified. + /// : an opaque cache validator surfaced verbatim as the + /// ETag response header, so whatever a client echoes back arrives here as + /// . It must change when retention removes items, not only when /// one is added, since nothing else tells a client its cached page is now wrong. /// - Task<(IList items, long total, string version)> GetEventLogItems( + Task>> GetEventLogItems( PagingInfo pagingInfo, string knownVersion = null); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence/Infrastructure/QueryResult.cs b/src/ServiceControl.Persistence/Infrastructure/QueryResult.cs index 3fb5568888..87e33fe962 100644 --- a/src/ServiceControl.Persistence/Infrastructure/QueryResult.cs +++ b/src/ServiceControl.Persistence/Infrastructure/QueryResult.cs @@ -11,8 +11,17 @@ public class QueryResult(TOut results, QueryStatsInfo queryStatsInfo) public QueryStatsInfo QueryStats { get; } = queryStatsInfo; + /// + /// The caller already holds this version, so was never fetched and is + /// null. is still populated. + /// + public bool NotModified { get; private init; } + public static QueryResult Empty() => new(null, QueryStatsInfo.Zero); + public static QueryResult Unchanged(QueryStatsInfo queryStatsInfo) => + new(null, queryStatsInfo) { NotModified = true }; + public static implicit operator Task>(QueryResult instance) => Task.FromResult(instance); } } \ No newline at end of file diff --git a/src/ServiceControl/EventLog/EventLogApiController.cs b/src/ServiceControl/EventLog/EventLogApiController.cs index 765b72fd0d..23c0a41dae 100644 --- a/src/ServiceControl/EventLog/EventLogApiController.cs +++ b/src/ServiceControl/EventLog/EventLogApiController.cs @@ -25,18 +25,17 @@ public async Task>> Items([FromQuery] Pagin var knownVersion = Request.Headers.IfNoneMatch.FirstOrDefault()?.Trim('"'); // Passing knownVersion lets the persister skip work it would otherwise waste - var (results, totalCount, version) = await logDataStore.GetEventLogItems(pagingInfo, knownVersion); + var result = await logDataStore.GetEventLogItems(pagingInfo, knownVersion); - Response.WithPagingLinksAndTotalCount(pagingInfo, totalCount); - Response.WithEtag(version); + Response.WithPagingLinksAndTotalCount(pagingInfo, result.QueryStats.TotalCount); + Response.WithEtag(result.QueryStats.ETag); - // Null items means "you already hold this version" - if (results is null) + if (result.NotModified) { return StatusCode((int)HttpStatusCode.NotModified); } - return Ok(results); + return Ok(result.Results); } } } \ No newline at end of file From 4fb0eb96ab02875a1e49015511c39ecf846e7275 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 29 Jul 2026 20:38:20 +0800 Subject: [PATCH 12/15] include HighestId in etag versioning and add unit test --- .../Implementation/EventLogDataStore.cs | 18 +++++++---- .../EFCore/EventLogDataStoreEFTests.cs | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index 170836af8b..a499bc5953 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -31,15 +31,20 @@ public Task>> GetEventLogItems( { var query = dbContext.EventLogItems.AsNoTracking(); - // Both aggregates in one round trip. Grouping on a constant collapses the table to a + // All three aggregates in one round trip. Grouping on a constant collapses the table to a // single row, and an empty table yields no rows at all, hence the null coalescing. var stats = await query .GroupBy(_ => 1) - .Select(g => new { Total = g.LongCount(), Newest = g.Max(e => (DateTime?)e.RaisedAt) }) + .Select(g => new + { + Total = g.LongCount(), + Newest = g.Max(e => (DateTime?)e.RaisedAt), + HighestId = g.Max(e => (long?)e.Id) + }) .FirstOrDefaultAsync(); var total = stats?.Total ?? 0; - var version = Version(total, stats?.Newest); + var version = Version(total, stats?.Newest, stats?.HighestId); var queryStats = new QueryStatsInfo(version, total, isStale: false); // The point of knownVersion. Everything above is index work. @@ -75,7 +80,8 @@ public Task>> GetEventLogItems( return new QueryResult>(items, queryStats); }); - // Synthesised version ID to be used for an ETag, using total count and the newest item's RaisedAt timestamp. - static string Version(long total, DateTime? newest) => - DeterministicGuid.MakeId($"{total}|{newest?.Ticks ?? 0}").ToString(); + // Synthesised version ID to be used for an ETag. The highest key is the monotonic term: identity + // values gap but never repeat, so an insert moves the version whatever its RaisedAt says. + static string Version(long total, DateTime? newest, long? highestId) => + DeterministicGuid.MakeId($"{total}|{newest?.Ticks ?? 0}|{highestId ?? 0}").ToString(); } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs index b9c93a7b1a..a6e4b563e6 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -94,6 +94,37 @@ public async Task Version_changes_when_rows_are_deleted_behind_the_interface() } } + [Test] + public async Task Version_changes_when_a_retention_delete_and_a_backdated_insert_cancel_out() + { + var baseTime = new DateTime(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc); + + for (var i = 0; i < 3; i++) + { + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); + } + + await CompleteDatabaseOperation(); + + var before = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + await DeleteOldest(); + + // Older than every surviving row, which is routine: RaisedAt is a domain timestamp, so a + // failure reported late carries an old value. + await EventLogDataStore.Add(CreateLogItem("Backdated", baseTime.AddYears(-1)), Guid.CreateVersion7()); + await CompleteDatabaseOperation(); + + var after = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(after.QueryStats.TotalCount, Is.EqualTo(before.QueryStats.TotalCount), "the setup only bites while the count is unchanged"); + Assert.That(after.Results.Max(i => i.RaisedAt), Is.EqualTo(before.Results.Max(i => i.RaisedAt)), "and while the newest RaisedAt is unchanged"); + Assert.That(after.QueryStats.ETag, Is.Not.EqualTo(before.QueryStats.ETag), "the page changed, so the validator must too"); + } + } + async Task DeleteOldest() { using var scope = ServiceProvider.CreateScope(); From 3461632901a24fa2746388a598f32ae69c47648b Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 30 Jul 2026 18:06:40 +0800 Subject: [PATCH 13/15] Remove global event id --- docs/testing.md | 2 +- ...en_the_event_log_is_polled_with_an_etag.cs | 2 +- ... 20260730090531_EventLogItems.Designer.cs} | 10 +--- ...ems.cs => 20260730090531_EventLogItems.cs} | 7 --- ...SqlServiceControlDbContextModelSnapshot.cs | 8 --- ... 20260730090510_EventLogItems.Designer.cs} | 8 +-- ...ems.cs => 20260730090510_EventLogItems.cs} | 7 --- ...verServiceControlDbContextModelSnapshot.cs | 6 -- .../Entities/EventLogItemEntity.cs | 7 +-- .../EventLogItemConfiguration.cs | 2 - .../Implementation/EventLogDataStore.cs | 28 ++++----- .../EventLogDataStore.cs | 7 ++- .../EventLogItemDocumentIdTests.cs | 57 +++++++++++++++++++ .../Expiration/MessageExpiryTests.cs | 3 +- .../EFCore/EventLogDataStoreEFTests.cs | 15 +++-- .../EFCore/RetentionSweepTests.cs | 1 - .../EventLogDataStoreTests.cs | 29 ++++++---- .../EventLog/EventLogItemView.cs | 2 +- .../IEventLogDataStore.cs | 10 +--- .../EventLog/AuditEventLogWriter.cs | 4 +- .../EventLog/EventLogApiController.cs | 7 +-- .../WebApi/HttpRequestExtensions.cs | 26 +++++++++ 22 files changed, 137 insertions(+), 111 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260729082915_EventLogItems.Designer.cs => 20260730090531_EventLogItems.Designer.cs} (97%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260729082915_EventLogItems.cs => 20260730090531_EventLogItems.cs} (86%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260729082911_EventLogItems.Designer.cs => 20260730090510_EventLogItems.Designer.cs} (97%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260729082911_EventLogItems.cs => 20260730090510_EventLogItems.cs} (85%) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemDocumentIdTests.cs create mode 100644 src/ServiceControl/Infrastructure/WebApi/HttpRequestExtensions.cs diff --git a/docs/testing.md b/docs/testing.md index 8ded9542b5..0d24541530 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -42,7 +42,7 @@ System.Security.SecurityException : The source ServiceControl was not found on c but some or all event logs could not be searched. Inaccessible logs: Security. ``` -Setup registers a Windows event source, and checking whether one already exists enumerates every event log — including `Security`, which a process without administrator rights cannot read. +Setup registers a Windows event source, and checking whether one already exists enumerates every event log, including `Security`, which a process without administrator rights cannot read. Register both sources once, from an **elevated** PowerShell prompt: diff --git a/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs b/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs index b3c758bfef..aef1816f6f 100644 --- a/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs +++ b/src/ServiceControl.AcceptanceTests/EventLogs/When_the_event_log_is_polled_with_an_etag.cs @@ -80,7 +80,7 @@ await Define() "a client holding the current version must be told so, not handed the page again"); Assert.That(unknownEtagStatus, Is.EqualTo(HttpStatusCode.OK), - "an unrecognised validator is a cache miss — this is what stops an unconditional 304 passing"); + "an unrecognised validator is a cache miss: this is what stops an unconditional 304 passing"); Assert.That(unknownEtagReturnedItems, Is.True, "a cache miss must carry the items, not an empty body with a 200"); diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs similarity index 97% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs index fecd33ee3b..6e3ec245b1 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260729082915_EventLogItems")] + [Migration("20260730090531_EventLogItems")] partial class EventLogItems { /// @@ -82,17 +82,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("severity"); - b.Property("UniqueEventId") - .HasColumnType("uuid") - .HasColumnName("unique_event_id"); - b.HasKey("Id") .HasName("pk_event_log_items"); - b.HasIndex("UniqueEventId") - .IsUnique() - .HasDatabaseName("ix_event_log_items_unique_event_id"); - b.HasIndex("RaisedAt", "Id") .IsDescending() .HasDatabaseName("ix_event_log_items_raised_at_id"); diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.cs similarity index 86% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.cs index e6c3037c98..5b7be41547 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260729082915_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.cs @@ -19,7 +19,6 @@ protected override void Up(MigrationBuilder migrationBuilder) { id = table.Column(type: "bigint", nullable: false) .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), - unique_event_id = table.Column(type: "uuid", nullable: false), description = table.Column(type: "text", nullable: false), severity = table.Column(type: "integer", nullable: false), raised_at = table.Column(type: "timestamp with time zone", nullable: false), @@ -37,12 +36,6 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "EventLogItems", columns: new[] { "raised_at", "id" }, descending: new bool[0]); - - migrationBuilder.CreateIndex( - name: "ix_event_log_items_unique_event_id", - table: "EventLogItems", - column: "unique_event_id", - unique: true); } /// diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 41b00e9307..3523cd529d 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -79,17 +79,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("severity"); - b.Property("UniqueEventId") - .HasColumnType("uuid") - .HasColumnName("unique_event_id"); - b.HasKey("Id") .HasName("pk_event_log_items"); - b.HasIndex("UniqueEventId") - .IsUnique() - .HasDatabaseName("ix_event_log_items_unique_event_id"); - b.HasIndex("RaisedAt", "Id") .IsDescending() .HasDatabaseName("ix_event_log_items_raised_at_id"); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs similarity index 97% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs index cc49c4f5cc..79eaa9fd08 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260729082911_EventLogItems")] + [Migration("20260730090510_EventLogItems")] partial class EventLogItems { /// @@ -71,14 +71,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Severity") .HasColumnType("int"); - b.Property("UniqueEventId") - .HasColumnType("uniqueidentifier"); - b.HasKey("Id"); - b.HasIndex("UniqueEventId") - .IsUnique(); - b.HasIndex("RaisedAt", "Id") .IsDescending(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.cs similarity index 85% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.cs index ba6f815e1c..eea4e40e35 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260729082911_EventLogItems.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.cs @@ -17,7 +17,6 @@ protected override void Up(MigrationBuilder migrationBuilder) { Id = table.Column(type: "bigint", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), - UniqueEventId = table.Column(type: "uniqueidentifier", nullable: false), Description = table.Column(type: "nvarchar(max)", nullable: false), Severity = table.Column(type: "int", nullable: false), RaisedAt = table.Column(type: "datetime2", nullable: false), @@ -35,12 +34,6 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "EventLogItems", columns: new[] { "RaisedAt", "Id" }, descending: new bool[0]); - - migrationBuilder.CreateIndex( - name: "IX_EventLogItems_UniqueEventId", - table: "EventLogItems", - column: "UniqueEventId", - unique: true); } /// diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index 59473326f4..50f22c1f54 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -68,14 +68,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Severity") .HasColumnType("int"); - b.Property("UniqueEventId") - .HasColumnType("uniqueidentifier"); - b.HasKey("Id"); - b.HasIndex("UniqueEventId") - .IsUnique(); - b.HasIndex("RaisedAt", "Id") .IsDescending(); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs index ced3dee9f9..a31207ba26 100644 --- a/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs +++ b/src/ServiceControl.Persistence.EFCore/Entities/EventLogItemEntity.cs @@ -4,13 +4,10 @@ namespace ServiceControl.Persistence.EFCore.Entities; public class EventLogItemEntity { - // The physical key only. Sequential so inserts stay at the tail of the index, and narrow so it - // is a cheap tiebreaker for RaisedAt paging. + // Sequential so inserts stay at the tail of the index, and narrow so it is a cheap tiebreaker + // for RaisedAt paging. Also what the API returns as EventLogItemView.Id, so it is store-local. public long Id { get; set; } - // A globally unique identity common between persisters (e.g. used in migrations). - public Guid UniqueEventId { get; set; } - public required string Description { get; set; } public Severity Severity { get; set; } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs index 6dbe6f19e7..8a49b8272d 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/EventLogItemConfiguration.cs @@ -11,8 +11,6 @@ public void Configure(EntityTypeBuilder builder) builder.ToTable("EventLogItems"); builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedOnAdd(); - builder.Property(e => e.UniqueEventId).IsRequired(); - builder.HasIndex(e => e.UniqueEventId).IsUnique(); builder.Property(e => e.Description).IsRequired(); builder.Property(e => e.Severity).IsRequired(); builder.Property(e => e.RaisedAt).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs index a499bc5953..11f8475337 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EventLogDataStore.cs @@ -8,12 +8,11 @@ namespace ServiceControl.Persistence.EFCore.Implementation; public class EventLogDataStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IEventLogDataStore { - public Task Add(EventLogItem logItem, Guid eventId) => + public Task Add(EventLogItem logItem) => ExecuteWithDbContext(async dbContext => { dbContext.EventLogItems.Add(new EventLogItemEntity { - UniqueEventId = eventId, Description = logItem.Description, Severity = logItem.Severity, RaisedAt = logItem.RaisedAt, @@ -55,28 +54,25 @@ public Task>> GetEventLogItems( return QueryResult>.Unchanged(queryStats); } - var rows = await query + var items = await query // The key breaks ties so that items sharing a RaisedAt cannot shuffle between // pages. IX_EventLogItems_RaisedAt_Id is declared in exactly this order. .OrderByDescending(e => e.RaisedAt) .ThenByDescending(e => e.Id) .Skip(pagingInfo.Offset) .Take(pagingInfo.PageSize) + .Select(e => new EventLogItemView + { + Id = e.Id.ToString(), + Description = e.Description, + Severity = e.Severity, + RaisedAt = e.RaisedAt, + RelatedTo = e.RelatedTo, + Category = e.Category, + EventType = e.EventType + }) .ToListAsync(); - // The id is stringified here, not in the query: SQL Server converts uniqueidentifier - // to uppercase hex, while Guid.ToString() and PostgreSQL both produce lowercase. - var items = rows.Select(e => new EventLogItemView - { - Id = e.UniqueEventId.ToString(), - Description = e.Description, - Severity = e.Severity, - RaisedAt = e.RaisedAt, - RelatedTo = e.RelatedTo, - Category = e.Category, - EventType = e.EventType - }).ToList(); - return new QueryResult>(items, queryStats); }); diff --git a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs index a1ef4c5bf1..656fb45b04 100644 --- a/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EventLogDataStore.cs @@ -9,12 +9,15 @@ class EventLogDataStore(IRavenSessionProvider sessionProvider, ExpirationManager expirationManager) : IEventLogDataStore { - public async Task Add(EventLogItem logItem, Guid eventId) + public async Task Add(EventLogItem logItem) { using var session = await sessionProvider.OpenSession(); + + // Version 7 rather than a random GUID so the final segment is time-ordered, which keeps + // documents written together adjacent in the id index. await session.StoreAsync( logItem, - EventLogItemIdGenerator.MakeDocumentId(logItem.Category, logItem.EventType, eventId)); + EventLogItemIdGenerator.MakeDocumentId(logItem.Category, logItem.EventType, Guid.CreateVersion7())); // Retention on RavenDB is per-document expiry metadata stamped at write time, not a // sweep. It has to be set here, on the only write path, or items never expire. diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemDocumentIdTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemDocumentIdTests.cs new file mode 100644 index 0000000000..5b682bd499 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DocumentIdGenerators/EventLogItemDocumentIdTests.cs @@ -0,0 +1,57 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DocumentIdGenerators +{ + using System; + using System.Threading.Tasks; + using NUnit.Framework; + using ServiceControl.EventLog; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.Persistence.RavenDB; + + [TestFixture] + class EventLogItemDocumentIdTests : RavenPersistenceTestBase + { + [Test] + public async Task Returned_id_is_a_document_id_for_the_items_category_and_event_type() + { + await EventLogDataStore.Add(Item("Message processing failed")); + await CompleteDatabaseOperation(); + + var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; + + var id = items[0].Id; + using (Assert.EnterMultipleScope()) + { + Assert.That(id, Does.StartWith("EventLogItem/Recoverability/MessageFailed/")); + Assert.That( + EventLogItemIdGenerator.GetEventIdFromDocumentId(id), + Is.Not.EqualTo(Guid.Empty), + "the final segment must be the minted event id, not an empty or absent GUID"); + } + } + + // Category and EventType are identical here, so the minted GUID is the only thing keeping the + // two document ids apart. Were it dropped or made deterministic, the second write would + // silently replace the first rather than adding to the feed. + [Test] + public async Task Two_items_of_the_same_kind_do_not_overwrite_each_other() + { + await EventLogDataStore.Add(Item("first")); + await EventLogDataStore.Add(Item("second")); + await CompleteDatabaseOperation(); + + var result = await EventLogDataStore.GetEventLogItems(new PagingInfo()); + + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(2)); + } + + static EventLogItem Item(string description) => new() + { + Category = "Recoverability", + EventType = "MessageFailed", + Description = description, + Severity = Severity.Error, + RaisedAt = new DateTime(2026, 7, 22, 10, 30, 0, DateTimeKind.Utc), + RelatedTo = [] + }; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs index 07daacb443..58d36c02d3 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/Expiration/MessageExpiryTests.cs @@ -247,8 +247,7 @@ public async Task EventLogItemShouldExpire() await DisableExpiration(); await EventLogDataStore.Add( - new EventLogItem { Category = "Recoverability", EventType = "MessageFailed" }, - Guid.CreateVersion7()); + new EventLogItem { Category = "Recoverability", EventType = "MessageFailed" }); await CompleteDatabaseOperation(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs index a6e4b563e6..34b98aada7 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EventLogDataStoreEFTests.cs @@ -24,9 +24,8 @@ public async Task Items_sharing_a_raised_at_page_without_overlap_or_omission() for (var i = 0; i < 10; i++) { - var eventId = Guid.CreateVersion7(); - expected.Add(eventId.ToString()); - await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant), eventId); + expected.Add($"Collision{i}"); + await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant)); } await CompleteDatabaseOperation(); @@ -35,7 +34,7 @@ public async Task Items_sharing_a_raised_at_page_without_overlap_or_omission() for (var page = 1; page <= 5; page++) { var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo(page: page, pageSize: 2))).Results; - paged.AddRange(items.Select(i => i.Id)); + paged.AddRange(items.Select(i => i.EventType)); } using (Assert.EnterMultipleScope()) @@ -53,7 +52,7 @@ public async Task Paging_order_is_repeatable_when_raised_at_collides() for (var i = 0; i < 6; i++) { - await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem($"Collision{i}", sameInstant)); } await CompleteDatabaseOperation(); @@ -73,7 +72,7 @@ public async Task Version_changes_when_rows_are_deleted_behind_the_interface() for (var i = 0; i < 3; i++) { - await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); } await CompleteDatabaseOperation(); @@ -101,7 +100,7 @@ public async Task Version_changes_when_a_retention_delete_and_a_backdated_insert for (var i = 0; i < 3; i++) { - await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); } await CompleteDatabaseOperation(); @@ -112,7 +111,7 @@ public async Task Version_changes_when_a_retention_delete_and_a_backdated_insert // Older than every surviving row, which is routine: RaisedAt is a domain timestamp, so a // failure reported late carries an old value. - await EventLogDataStore.Add(CreateLogItem("Backdated", baseTime.AddYears(-1)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem("Backdated", baseTime.AddYears(-1))); await CompleteDatabaseOperation(); var after = await EventLogDataStore.GetEventLogItems(new PagingInfo()); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 53a4104bf3..5a2939e672 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -202,7 +202,6 @@ public async Task Sweeping_event_log_items_changes_the_version() static EventLogItemEntity EventLogRow(string marker, DateTime raisedAt) => new() { - UniqueEventId = Guid.CreateVersion7(), Description = marker, Severity = Severity.Info, RaisedAt = raisedAt, diff --git a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs index bc93896573..5b955d05d2 100644 --- a/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EventLogDataStoreTests.cs @@ -18,7 +18,7 @@ public async Task Added_item_is_returned_with_all_values_intact() logItem.Description = "Message processing failed"; logItem.RelatedTo = ["/message/abc123", "/endpoint/Sales"]; - await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); + await EventLogDataStore.Add(logItem); await CompleteDatabaseOperation(); var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; @@ -37,18 +37,23 @@ public async Task Added_item_is_returned_with_all_values_intact() } [Test] - public async Task The_supplied_event_id_is_recoverable_from_the_stored_item() + public async Task Every_returned_item_carries_its_own_id() { - var eventId = Guid.CreateVersion7(); - var logItem = CreateLogItem("MessageFailed", new DateTime(2026, 7, 22, 10, 30, 0, DateTimeKind.Utc)); + var raisedAt = new DateTime(2026, 7, 22, 10, 30, 0, DateTimeKind.Utc); - await EventLogDataStore.Add(logItem, eventId); + await EventLogDataStore.Add(CreateLogItem("MessageFailed", raisedAt)); + await EventLogDataStore.Add(CreateLogItem("EndpointStarted", raisedAt)); await CompleteDatabaseOperation(); var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; - // Not equality: Different persisters may return the event id in different formats. - Assert.That(items[0].Id, Does.Contain(eventId.ToString())); + Assert.That(items, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(items[0].Id, Is.Not.Null.And.Not.Empty); + Assert.That(items[1].Id, Is.Not.Null.And.Not.Empty); + Assert.That(items[0].Id, Is.Not.EqualTo(items[1].Id)); + } } [Test] @@ -57,7 +62,7 @@ public async Task Item_with_no_related_links_round_trips_as_empty() var logItem = CreateLogItem("EndpointStarted", DateTime.UtcNow); logItem.RelatedTo = []; - await EventLogDataStore.Add(logItem, Guid.CreateVersion7()); + await EventLogDataStore.Add(logItem); await CompleteDatabaseOperation(); var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; @@ -69,9 +74,9 @@ public async Task Item_with_no_related_links_round_trips_as_empty() public async Task Items_are_returned_most_recently_raised_first() { var baseTime = new DateTime(2026, 7, 22, 9, 0, 0, DateTimeKind.Utc); - await EventLogDataStore.Add(CreateLogItem("Oldest", baseTime), Guid.CreateVersion7()); - await EventLogDataStore.Add(CreateLogItem("Newest", baseTime.AddMinutes(2)), Guid.CreateVersion7()); - await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem("Oldest", baseTime)); + await EventLogDataStore.Add(CreateLogItem("Newest", baseTime.AddMinutes(2))); + await EventLogDataStore.Add(CreateLogItem("Middle", baseTime.AddMinutes(1))); await CompleteDatabaseOperation(); var items = (await EventLogDataStore.GetEventLogItems(new PagingInfo())).Results; @@ -257,7 +262,7 @@ async Task AddItems(int count) for (var i = 0; i < count; i++) { - await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i)), Guid.CreateVersion7()); + await EventLogDataStore.Add(CreateLogItem($"Event{i}", baseTime.AddMinutes(i))); } await CompleteDatabaseOperation(); diff --git a/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs b/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs index 5f6bf787f0..c3646ac8fc 100644 --- a/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs +++ b/src/ServiceControl.Persistence/EventLog/EventLogItemView.cs @@ -9,7 +9,7 @@ namespace ServiceControl.EventLog public class EventLogItemView { /// - /// Assigned by whichever persister stored the item + /// Assigned by whichever persister stored the item, and opaque. /// public string Id { get; set; } public string Description { get; set; } diff --git a/src/ServiceControl.Persistence/IEventLogDataStore.cs b/src/ServiceControl.Persistence/IEventLogDataStore.cs index 34ff1a9d48..09433bce1e 100644 --- a/src/ServiceControl.Persistence/IEventLogDataStore.cs +++ b/src/ServiceControl.Persistence/IEventLogDataStore.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Persistence { - using System; using System.Collections.Generic; using System.Threading.Tasks; using EventLog; @@ -17,14 +16,11 @@ public interface IEventLogDataStore { /// - /// Persists a single event log item. + /// Persists a single event log item. Identity is the persister's to assign, and is surfaced + /// on when the item is read back. /// /// The item to store. - /// - /// The item's portable/global identity, minted by the caller, and survives a - /// move between persisters. - /// - Task Add(EventLogItem logItem, Guid eventId); + Task Add(EventLogItem logItem); /// /// Returns one page of event log items, newest RaisedAt first. diff --git a/src/ServiceControl/EventLog/AuditEventLogWriter.cs b/src/ServiceControl/EventLog/AuditEventLogWriter.cs index 69e15f78f4..aeb14eafc9 100644 --- a/src/ServiceControl/EventLog/AuditEventLogWriter.cs +++ b/src/ServiceControl/EventLog/AuditEventLogWriter.cs @@ -1,6 +1,5 @@ namespace ServiceControl.EventLog { - using System; using System.Threading; using System.Threading.Tasks; using Infrastructure.DomainEvents; @@ -26,9 +25,8 @@ public async Task Handle(IDomainEvent message, CancellationToken cancellationTok } var logItem = mappings.ApplyMapping(message); - var eventId = Guid.CreateVersion7(); - await dataStore.Add(logItem, eventId); + await dataStore.Add(logItem); } readonly IEventLogDataStore dataStore; diff --git a/src/ServiceControl/EventLog/EventLogApiController.cs b/src/ServiceControl/EventLog/EventLogApiController.cs index 23c0a41dae..4fdb061aec 100644 --- a/src/ServiceControl/EventLog/EventLogApiController.cs +++ b/src/ServiceControl/EventLog/EventLogApiController.cs @@ -1,7 +1,6 @@ namespace ServiceControl.EventLog { using System.Collections.Generic; - using System.Linq; using System.Net; using System.Threading.Tasks; using Infrastructure.Auth; @@ -20,12 +19,8 @@ public class EventLogApiController(IEventLogDataStore logDataStore) : Controller [HttpGet] public async Task>> Items([FromQuery] PagingInfo pagingInfo) { - - // The Trim handles both ETag formats (quoted and unquoted) deliberately. - var knownVersion = Request.Headers.IfNoneMatch.FirstOrDefault()?.Trim('"'); - // Passing knownVersion lets the persister skip work it would otherwise waste - var result = await logDataStore.GetEventLogItems(pagingInfo, knownVersion); + var result = await logDataStore.GetEventLogItems(pagingInfo, Request.GetKnownVersion()); Response.WithPagingLinksAndTotalCount(pagingInfo, result.QueryStats.TotalCount); Response.WithEtag(result.QueryStats.ETag); diff --git a/src/ServiceControl/Infrastructure/WebApi/HttpRequestExtensions.cs b/src/ServiceControl/Infrastructure/WebApi/HttpRequestExtensions.cs new file mode 100644 index 0000000000..f4210fde5a --- /dev/null +++ b/src/ServiceControl/Infrastructure/WebApi/HttpRequestExtensions.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Infrastructure.WebApi +{ + using System.Linq; + using Microsoft.AspNetCore.Http; + + static class HttpRequestExtensions + { + /// + /// The validator the caller already holds, unquoted so it can be compared against a store's + /// own version, or null if the caller holds none. + /// + /// Only meaningful for an endpoint that publishes its validator through + /// . An endpoint publishing through + /// WithDeterministicEtag hashes the validator on the way out, so what a client echoes + /// back cannot be compared with anything a store holds and this would never match. + /// + /// + public static string GetKnownVersion(this HttpRequest request) => + Unquote(request.Headers.IfNoneMatch.FirstOrDefault()); + + // Trimming every quote instead would turn a malformed header + // into a truncated value rather than into the cache miss it should be. + static string Unquote(string etag) => + etag?.Length > 1 && etag[0] == '"' && etag[^1] == '"' ? etag[1..^1] : etag; + } +} From f1698db32f5060b0a79c8af2b37be8d38c3c656e Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 31 Jul 2026 08:56:59 +0800 Subject: [PATCH 14/15] Add eventlogs design --- docs/eventlog-design.md | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/eventlog-design.md diff --git a/docs/eventlog-design.md b/docs/eventlog-design.md new file mode 100644 index 0000000000..17c01c2110 --- /dev/null +++ b/docs/eventlog-design.md @@ -0,0 +1,55 @@ +# Event log design + +## What it is + +The event log is the primary instance's activity feed: the chronological "what has this instance noticed" list that ServicePulse shows. Message failures, retries, redirects, heartbeats, custom checks and integration failures all surface here. + +It is a **projection of domain events, not a log file**. Nothing writes to it directly. Components raise domain events for their own reasons, and the event log turns a chosen subset of those into feed items. An event only appears if someone has declared how it should read, which makes the feed an editorial selection rather than a dump. + +Only the primary instance has an event log. Audit and monitoring instances have none. + +Four contracts define the whole part: `EventLogItem` (what is written), `EventLogItemView` (what is read), `EventLogMappingDefinition` (how a component declares an event belongs in the feed), and `IEventLogDataStore` (the storage seam). Everything else is machinery behind them. + +## What gets recorded + +`EventLogItem` is the write contract, and it carries **no identity**. The identity is minted in each persistence seam. + +`EventLogItemView` is the same shape plus `Id`, which is assigned by storage rather than by the application. + +`RelatedTo` entries are built by helpers with fixed prefixes: `/message/{id}`, `/endpoint/{name}`, `/machine/{name}`, `/host/{guid}`, `/customcheck/{id}`, `/recoverability/groups/{id}`. The helpers only prefix a string, so passing the wrong property yields a link that resolves to nothing. + +## Declaring an event + +A component makes one of its events visible with two things, and nothing in the event log changes: + +1. A class deriving **directly** from `EventLogMappingDefinition`. Derive through an intermediate non-generic base and the definition is silently skipped at registration. +2. A matching `services.AddEventLogMapping()` in that component's configuration. Two definitions for one event type is an error. + +A definition is a **declarative builder, not a handler**: its constructor calls `Description(…)`, and optionally `RaisedAt(…)`, `Severity(…)` / `TreatAsError()` and the `RelatesTo*` helpers, to specify how one row reads. Definitions live with the component that raises the event: **this part owns the machinery, the components own the content**. An event with no declaration is ignored, deliberately and silently. + +## Reading the feed + +`GET /api/eventlogitems` is the entire HTTP surface, gated on `Permissions.ErrorEventLogView`. It takes `page` and `pageSize`, returns one page of `EventLogItemView` newest first, and sets `Total-Count`, `ETag` and `Link`. There is no write, no delete, no per-item lookup, and no filtering or search: `Category`, `Severity` and `EventType` are returned but cannot be queried on. + +Clients discover new items by **polling**; nothing is pushed, and recording an item has no outward effect at all. Because polling is the only path, a caller that echoes its `ETag` back as `If-None-Match` gets `304` with no body, and the request costs a header exchange instead of a page of JSON. + +Timestamps are when the thing happened, so an item can land in the middle of the feed rather than at its head. + +## The storage seam + +`IEventLogDataStore` has two methods, and its XML docs are the binding contract: + +- `Add(EventLogItem)` persists one item. **Identity is the store's to assign** and surfaces on `EventLogItemView.Id`. That makes `Id` opaque: a stable key within one store, not something to parse. +- `GetEventLogItems(PagingInfo, knownVersion)` returns a `QueryResult` carrying the page, the total count independent of paging, and an `ETag`. Two obligations: the `ETag` is surfaced **verbatim**, so whatever the client echoes back arrives here unchanged and can be compared, and it **must change when retention removes items**, not only when one is added, since nothing else tells a polling client its cached page has gone stale. + +## Retention + +Items age out on their own after `EventsRetentionPeriod`, 14 days by default. Nothing a user does removes one and there is no API to try. Enforcement is left to each storage backend and is invisible through the seam, so a change to the setting applies retrospectively on some backends and to new items only on others. + +## Failure behaviour + +Recording is **in-band, not best-effort**. Domain event dispatch rethrows, so if storage is unreachable the failure propagates to whatever raised the event and that operation fails. There is no retry, no queue and no buffer in front of the write. + +## Known limits + +- **The feed is per error instance.** A federated deployment gets no aggregation and nothing in the response says which instance answered. From c77f1023faa3b60354c8b48c484c7e4c29a93825 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 31 Jul 2026 09:21:05 +0800 Subject: [PATCH 15/15] Regenerate eventlog migrations --- ... 20260731010357_EventLogItems.Designer.cs} | 26 ++++++++++++++++++- ...ems.cs => 20260731010357_EventLogItems.cs} | 0 ... 20260731010345_EventLogItems.Designer.cs} | 22 +++++++++++++++- ...ems.cs => 20260731010345_EventLogItems.cs} | 0 4 files changed, 46 insertions(+), 2 deletions(-) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260730090531_EventLogItems.Designer.cs => 20260731010357_EventLogItems.Designer.cs} (94%) rename src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/{20260730090531_EventLogItems.cs => 20260731010357_EventLogItems.cs} (100%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260730090510_EventLogItems.Designer.cs => 20260731010345_EventLogItems.Designer.cs} (93%) rename src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/{20260730090510_EventLogItems.cs => 20260731010345_EventLogItems.cs} (100%) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260731010357_EventLogItems.Designer.cs similarity index 94% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260731010357_EventLogItems.Designer.cs index 6e3ec245b1..a6ade62c6a 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260731010357_EventLogItems.Designer.cs @@ -13,7 +13,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations { [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260730090531_EventLogItems")] + [Migration("20260731010357_EventLogItems")] partial class EventLogItems { /// @@ -369,6 +369,30 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("known_endpoints", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.TrialMetadataEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260731010357_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260730090531_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260731010357_EventLogItems.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260731010345_EventLogItems.Designer.cs similarity index 93% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260731010345_EventLogItems.Designer.cs index 79eaa9fd08..4b16d60ff9 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.Designer.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260731010345_EventLogItems.Designer.cs @@ -12,7 +12,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations { [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260730090510_EventLogItems")] + [Migration("20260731010345_EventLogItems")] partial class EventLogItems { /// @@ -296,6 +296,26 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("KnownEndpoints"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.TrialMetadataEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260731010345_EventLogItems.cs similarity index 100% rename from src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260730090510_EventLogItems.cs rename to src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260731010345_EventLogItems.cs