From 281071eb86b7e3df39be4c85d2ef16334ac50a58 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 10 Jul 2026 07:38:28 -0500 Subject: [PATCH 1/9] fix: harden cleanup jobs and repository operations --- src/Exceptionless.Core/Jobs/CleanupDataJob.cs | 19 +- .../Jobs/CleanupOrphanedDataJob.cs | 365 +++++++----------- .../Repositories/EventRepository.cs | 189 ++++++++- .../Interfaces/IEventRepository.cs | 8 + .../Interfaces/IStackRepository.cs | 1 + .../Repositories/StackRepository.cs | 16 + .../Exceptionless.Web.csproj | 1 + .../Jobs/CleanupDataJobPaginationTests.cs | 160 ++++++++ .../Jobs/CleanupDataJobTests.cs | 17 +- .../Jobs/CleanupOrphanedDataJobTests.cs | 331 ++++++++++++---- .../Repositories/EventRepositoryTests.cs | 182 +++++++++ .../Repositories/StackRepositoryTests.cs | 56 +++ 12 files changed, 1028 insertions(+), 317 deletions(-) create mode 100644 tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index f212c67cf4..65ebc19597 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -86,7 +86,7 @@ ILoggerFactory loggerFactory protected override Task GetLockAsync(CancellationToken cancellationToken = default) { - return _lockProvider.TryAcquireAsync(nameof(CleanupDataJob), TimeSpan.FromMinutes(15), cancellationToken); + return _lockProvider.TryAcquireAsync(nameof(CleanupDataJob), TimeSpan.FromHours(2), cancellationToken); } protected override async Task RunInternalAsync(JobContext context) @@ -201,8 +201,13 @@ private async Task CleanupSoftDeletedOrganizationsAsync(JobContext context) while (organizationResults.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var organization in organizationResults.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); try { @@ -229,8 +234,13 @@ private async Task CleanupSoftDeletedProjectsAsync(JobContext context) while (projectResults.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var project in projectResults.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id)); try { @@ -383,8 +393,13 @@ private async Task EnforceRetentionAsync(JobContext context) var results = await _organizationRepository.FindAsync(q => q.Include(o => o.Id, o => o.Name, o => o.RetentionDays), o => o.SearchAfterPaging().PageLimit(100)); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var organization in results.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); int retentionDays = _billingManager.GetBillingPlanByUpsellingRetentionPeriod(organization.RetentionDays)?.RetentionDays ?? _appOptions.MaximumRetentionDays; @@ -454,6 +469,8 @@ private async Task EnforceEventRetentionDaysAsync(Organization organization, int private Task RenewLockAsync(JobContext context) { + // Called at each page boundary to prevent the distributed lock from expiring + // during long-running bulk cleanup operations that span multiple pages. _lastRun = _timeProvider.GetUtcNow().UtcDateTime; return context.RenewLockAsync(); } diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 37a67ae24a..5e6e4bc257 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -1,51 +1,42 @@ -using Elastic.Clients.Elasticsearch; -using Elastic.Clients.Elasticsearch.Aggregations; -using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; -using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Configuration; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Repositories; -using Foundatio.Repositories.Elasticsearch.Extensions; -using Foundatio.Repositories.Elasticsearch.Utility; -using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Resilience; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Jobs; [Job(Description = "Deletes orphaned data.", IsContinuous = false)] public class CleanupOrphanedDataJob : JobWithLockBase, IHealthCheck { - private readonly ExceptionlessElasticConfiguration _config; - private readonly ElasticsearchClient _elasticClient; - private readonly IStackRepository _stackRepository; - private readonly IProjectRepository _projectRepository; private readonly IOrganizationRepository _organizationRepository; + private readonly IProjectRepository _projectRepository; + private readonly IStackRepository _stackRepository; private readonly IEventRepository _eventRepository; private readonly ICacheClient _cacheClient; private readonly ILockProvider _lockProvider; private DateTime? _lastRun; - public CleanupOrphanedDataJob(ExceptionlessElasticConfiguration config, IStackRepository stackRepository, - IProjectRepository projectRepository, IOrganizationRepository organizationRepository, - IEventRepository eventRepository, ICacheClient cacheClient, ILockProvider lockProvider, + public CleanupOrphanedDataJob( + IOrganizationRepository organizationRepository, + IProjectRepository projectRepository, + IStackRepository stackRepository, + IEventRepository eventRepository, + ICacheClient cacheClient, + ILockProvider lockProvider, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory ) : base(timeProvider, resiliencePolicyProvider, loggerFactory) { - _config = config; - _elasticClient = config.Client; - _stackRepository = stackRepository; - _projectRepository = projectRepository; _organizationRepository = organizationRepository; + _projectRepository = projectRepository; + _stackRepository = stackRepository; _eventRepository = eventRepository; _cacheClient = cacheClient; _lockProvider = lockProvider; @@ -58,6 +49,8 @@ ILoggerFactory loggerFactory protected override async Task RunInternalAsync(JobContext context) { + _lastRun = _timeProvider.GetUtcNow().UtcDateTime; + await DeleteOrphanedEventsByStackAsync(context); await DeleteOrphanedEventsByProjectAsync(context); await DeleteOrphanedEventsByOrganizationAsync(context); @@ -69,200 +62,154 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { - // get approximate number of unique stack ids - var stackCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); - - double? uniqueStackIdCount = stackCardinality.Aggregations?.GetCardinality("cardinality_stack_id")?.Value; - if (!uniqueStackIdCount.HasValue || uniqueStackIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueStackIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalStackIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by stack"); + long totalOrphanedEvents = 0; + long totalStackIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var stackIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - - string[] stackIds = stackIdTerms.Aggregations?.GetStringTerms("terms_stack_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (stackIds.Length == 0) - continue; + var page = await _eventRepository.GetDistinctStackIdsAsync(500, nextValue, context.CancellationToken); + var stackIds = page.Values; + if (stackIds.Count == 0) + break; - totalStackIds += stackIds.Length; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalStackIds += stackIds.Count; - var stacks = await _stackRepository.GetByIdsAsync(stackIds, o => o.ImmediateConsistency()); - var foundStackIds = stacks.Select(stack => stack.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingStackIds = stackIds.Where(stackId => !foundStackIds.Contains(stackId)).ToArray(); + var existingStacks = await _stackRepository.GetByIdsAsync(stackIds.ToArray(), o => o.Include(s => s.Id, s => s.IsDeleted)); + var existingStackIds = existingStacks.Select(s => s.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingStackIds = stackIds.Where(id => !existingStackIds.Contains(id)).ToArray(); if (missingStackIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing stacks out of {StackIdCount}", batchNumber, buckets, stackIds.Length); continue; - } - totalOrphanedEventCount += missingStackIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(missingStackIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + long deletedCount = await _eventRepository.RemoveAllByStackIdsAsync(missingStackIds); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingStackCount} missing stacks out of {StackIdCount} checked", deletedCount, missingStackIds.Length, stackIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount}", totalOrphanedEventCount, totalStackIds); + _logger.LogInformation("Completed orphaned events cleanup by stack: deleted {TotalOrphanedEvents} events, checked {TotalStackIds} stacks", totalOrphanedEvents, totalStackIds); } public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { - // get approximate number of unique project ids - var projectCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); - - double? uniqueProjectIdCount = projectCardinality.Aggregations?.GetCardinality("cardinality_project_id")?.Value; - if (!uniqueProjectIdCount.HasValue || uniqueProjectIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueProjectIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalProjectIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by project"); + long totalOrphanedEvents = 0; + long totalProjectIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var projectIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); + var page = await _eventRepository.GetDistinctProjectIdsAsync(500, nextValue, context.CancellationToken); + var projectIds = page.Values; + if (projectIds.Count == 0) + break; - string[] projectIds = projectIdTerms.Aggregations?.GetStringTerms("terms_project_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (projectIds.Length == 0) - continue; - - totalProjectIds += projectIds.Length; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalProjectIds += projectIds.Count; - var projects = await _projectRepository.GetByIdsAsync(projectIds, o => o.ImmediateConsistency()); - var foundProjectIds = projects.Select(project => project.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingProjectIds = projectIds.Where(projectId => !foundProjectIds.Contains(projectId)).ToArray(); + var existingProjects = await _projectRepository.GetByIdsAsync(projectIds.ToArray(), o => o.Include(p => p.Id, p => p.IsDeleted)); + var existingProjectIds = existingProjects.Select(p => p.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingProjectIds = projectIds.Where(id => !existingProjectIds.Contains(id)).ToArray(); if (missingProjectIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing projects out of {ProjectIdCount}", batchNumber, buckets, projectIds.Length); continue; - } - totalOrphanedEventCount += missingProjectIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + long deletedCount = await _eventRepository.RemoveAllByProjectIdsAsync(missingProjectIds); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingProjectCount} missing projects out of {ProjectIdCount} checked", deletedCount, missingProjectIds.Length, projectIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount}", totalOrphanedEventCount, totalProjectIds); + _logger.LogInformation("Completed orphaned events cleanup by project: deleted {TotalOrphanedEvents} events, checked {TotalProjectIds} projects", totalOrphanedEvents, totalProjectIds); } public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { - // get approximate number of unique organization ids - var organizationCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); - - double? uniqueOrganizationIdCount = organizationCardinality.Aggregations?.GetCardinality("cardinality_organization_id")?.Value; - if (!uniqueOrganizationIdCount.HasValue || uniqueOrganizationIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueOrganizationIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalOrganizationIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by organization"); + long totalOrphanedEvents = 0; + long totalOrganizationIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var organizationIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); + var page = await _eventRepository.GetDistinctOrganizationIdsAsync(500, nextValue, context.CancellationToken); + var organizationIds = page.Values; + if (organizationIds.Count == 0) + break; - string[] organizationIds = organizationIdTerms.Aggregations?.GetStringTerms("terms_organization_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (organizationIds.Length == 0) - continue; - - totalOrganizationIds += organizationIds.Length; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalOrganizationIds += organizationIds.Count; - var organizations = await _organizationRepository.GetByIdsAsync(organizationIds, o => o.ImmediateConsistency()); - var foundOrganizationIds = organizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingOrganizationIds = organizationIds.Where(organizationId => !foundOrganizationIds.Contains(organizationId)).ToArray(); + var existingOrganizations = await _organizationRepository.GetByIdsAsync(organizationIds.ToArray(), o => o.Include(organization => organization.Id, organization => organization.IsDeleted)); + var existingOrganizationIds = existingOrganizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingOrganizationIds = organizationIds.Where(id => !existingOrganizationIds.Contains(id)).ToArray(); if (missingOrganizationIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing organizations out of {OrganizationIdCount}", batchNumber, buckets, organizationIds.Length); continue; - } - totalOrphanedEventCount += missingOrganizationIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + long deletedCount = await _eventRepository.RemoveAllByOrganizationIdsAsync(missingOrganizationIds); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingOrganizationCount} missing organizations out of {OrganizationIdCount} checked", deletedCount, missingOrganizationIds.Length, organizationIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount}", totalOrphanedEventCount, totalOrganizationIds); + _logger.LogInformation("Completed orphaned events cleanup by organization: deleted {TotalOrphanedEvents} events, checked {TotalOrganizationIds} organizations", totalOrphanedEvents, totalOrganizationIds); } public async Task FixDuplicateStacks(JobContext context) { _logger.LogInformation("Getting duplicate stacks"); - var duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .Indices(_config.Stacks.VersionedName) - .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) - .Size(0) - .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); - _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - - var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; - int total = buckets.Count; + int total = 0; int processed = 0; int error = 0; long totalUpdatedEventCount = 0; var lastStatus = _timeProvider.GetUtcNow().UtcDateTime; - int batch = 1; + int batch = 0; - while (buckets.Count > 0) + // Loop until no more duplicate signatures exist. Each iteration forces an index refresh + // (via ImmediateConsistency on GetDuplicateSignaturesAsync) so soft-deleted stacks are + // excluded from subsequent aggregation calls, preventing re-processing. + while (!context.CancellationToken.IsCancellationRequested) { - _logger.LogInformation($"Found {buckets.Count} duplicate stacks in batch #{batch}."); + var duplicateSignatures = await _stackRepository.GetDuplicateSignaturesAsync(); + if (duplicateSignatures.Count == 0) + break; + + batch++; + total += duplicateSignatures.Count; + _logger.LogInformation("Found {Total} duplicate stacks in batch #{Batch}", duplicateSignatures.Count, batch); await RenewLockAsync(context); - foreach (var duplicateSignature in buckets) + int batchProcessed = 0; + foreach (var duplicateSignature in duplicateSignatures) { + if (context.CancellationToken.IsCancellationRequested) + break; + string? projectId = null; string? signature = null; try { - string[] parts = duplicateSignature.Key.ToString().Split(':'); + string[] parts = duplicateSignature.Split(':'); if (parts.Length != 2) { - _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key.ToString()); + _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature); continue; } projectId = parts[0]; @@ -276,16 +223,16 @@ public async Task FixDuplicateStacks(JobContext context) } var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); - var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); + var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); - // we only need to update events if more than one stack has events associated to it + // We only need to update events if more than one stack has events associated to it. bool shouldUpdateEvents = eventCountBuckets.Count > 1; - // default to using the oldest stack + // Default to using the oldest stack. var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First(); var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList(); - // use the stack that has the most events on it so we can reduce the number of updates + // Use the stack that has the most events on it so we can reduce the number of updates. if (eventCountBuckets.Count > 0) { string targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key; @@ -297,102 +244,65 @@ public async Task FixDuplicateStacks(JobContext context) targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open; targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence); targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc); - targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed); ; + targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed); targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences); - targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags)); + targetStack.Tags.UnionWith(duplicateStacks.SelectMany(d => d.Tags)); targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList(); targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical); duplicateStacks.ForEach(s => s.IsDeleted = true); - await _stackRepository.SaveAsync(duplicateStacks); - await _stackRepository.SaveAsync(targetStack); - processed++; - - long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; - _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); if (shouldUpdateEvents) { - var response = await _elasticClient.UpdateByQueryAsync(u => u - .Indices(GetEventIndexPattern()) - .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => FieldValueHelper.ToFieldValue(s.Id)).ToList()))) - ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) - .Conflicts(Conflicts.Proceed) - .WaitForCompletion(false)); - _logger.LogRequest(response, LogLevel.Trace); - - var taskStartedTime = _timeProvider.GetUtcNow().UtcDateTime; - var taskId = response.Task; - int attempts = 0; - long affectedRecords = 0; - do - { - attempts++; - var taskStatus = await _elasticClient.Tasks.GetAsync(taskId!.FullyQualifiedId); - var status = taskStatus.Task.Status as ReindexStatus; - if (taskStatus.Completed) - { - // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - - affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); - break; - } - - if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - { - await RenewLockAsync(context); - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - } - - var delay = TimeSpan.FromMilliseconds(50); - if (attempts > 20) - delay = TimeSpan.FromSeconds(5); - else if (attempts > 10) - delay = TimeSpan.FromSeconds(1); - else if (attempts > 5) - delay = TimeSpan.FromMilliseconds(250); - - await Task.Delay(delay, _timeProvider, context.CancellationToken); - } while (true); + // Reassign events before soft-deleting duplicates: if event reassignment + // fails, the duplicate stacks remain visible and no data is lost. + long affectedRecords = await _eventRepository.ReassignStackAsync( + duplicateStacks.Select(s => s.Id), targetStack.Id); _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); - totalUpdatedEventCount += affectedRecords; } + // Soft-delete duplicates and save after events are safely migrated. + // No per-item ImmediateConsistency needed: GetDuplicateSignaturesAsync + // forces a refresh before each batch aggregation call. + await _stackRepository.SaveAsync([.. duplicateStacks, targetStack]); + processed++; + batchProcessed++; + + long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; + _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); + if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(lastStatus) > TimeSpan.FromSeconds(5)) { lastStatus = _timeProvider.GetUtcNow().UtcDateTime; + await RenewLockAsync(context); _logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); - await _cacheClient.RemoveByPrefixAsync(nameof(Stack)); } } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception ex) { + // Intentionally broad: log and continue processing other groups rather than + // aborting the entire job for a single corrupt or transiently failing signature. error++; - _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature); + _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}: {Message}", projectId, signature, ex.Message); } } - await _elasticClient.Indices.RefreshAsync(_config.Stacks.VersionedName); - duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .Indices(_config.Stacks.VersionedName) - .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) - .Size(0) - .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); - _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - - buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; - total += buckets.Count; - batch++; - - _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); + _logger.LogInformation("Batch #{Batch} complete: Processed={BatchProcessed} Total={Processed}/{Total} Errors={ErrorCount} UpdatedEvents={UpdatedEventCount}", batch, batchProcessed, processed, total, error, totalUpdatedEventCount); await _cacheClient.RemoveByPrefixAsync(nameof(Stack)); + + // If nothing was processed this batch (all errors), stop to avoid an infinite loop + // where the same failing signatures are retried indefinitely. + if (batchProcessed == 0) + break; } + + _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount} UpdatedEvents={UpdatedEventCount}", processed, total, error, totalUpdatedEventCount); } private Task RenewLockAsync(JobContext context) @@ -401,11 +311,6 @@ private Task RenewLockAsync(JobContext context) return context.RenewLockAsync(); } - private string GetEventIndexPattern() - { - return $"{_config.Events.VersionedName}-*"; - } - public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { if (!_lastRun.HasValue) diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 3fba37a69d..53fe18ec95 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -1,21 +1,30 @@ -using Elastic.Clients.Elasticsearch.QueryDsl; +using System.Linq.Expressions; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Transport; +using Elastic.Transport.Products.Elasticsearch; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Validation; using Exceptionless.DateTimeExtensions; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; namespace Exceptionless.Core.Repositories; public class EventRepository : RepositoryOwnedByOrganizationAndProject, IEventRepository { + private readonly ExceptionlessElasticConfiguration _configuration; private readonly TimeProvider _timeProvider; public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptions options, MiniValidationValidator validator) : base(configuration.Events, validator, options) { + _configuration = configuration; _timeProvider = configuration.TimeProvider; DisableCache(); // NOTE: If cache is ever enabled, then fast paths for patching/deleting with scripts will be super slow! @@ -74,7 +83,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, if (!String.IsNullOrEmpty(clientIpAddress)) query = query.FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); - return RemoveAllAsync(q => query, options); + return RemoveAllIgnoringMissingEventIndexesAsync(q => query, options); } public Task> GetByReferenceIdAsync(string projectId, string referenceId) @@ -188,12 +197,184 @@ public override Task> GetByProjectIdAsync(string pr return FindAsync(q => q.Project(projectId).SortDescending(e => e.Date).SortDescending(e => e.Id), options); } + public override Task RemoveAllByOrganizationIdAsync(string organizationId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId)); + } + + public override Task RemoveAllByProjectIdAsync(string organizationId, string projectId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + ArgumentException.ThrowIfNullOrEmpty(projectId); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId).Project(projectId)); + } + public Task RemoveAllByStackIdsAsync(string[] stackIds) { ArgumentNullException.ThrowIfNull(stackIds); - if (stackIds.Length == 0) + if (stackIds is []) throw new ArgumentOutOfRangeException(nameof(stackIds)); - return RemoveAllAsync(q => q.Stack(stackIds)); + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Stack(stackIds)); + } + + public Task RemoveAllByProjectIdsAsync(string[] projectIds) + { + ArgumentNullException.ThrowIfNull(projectIds); + if (projectIds is []) + throw new ArgumentOutOfRangeException(nameof(projectIds)); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Project(projectIds)); + } + + public Task RemoveAllByOrganizationIdsAsync(string[] organizationIds) + { + ArgumentNullException.ThrowIfNull(organizationIds); + if (organizationIds is []) + throw new ArgumentOutOfRangeException(nameof(organizationIds)); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationIds)); + } + + private async Task RemoveAllIgnoringMissingEventIndexesAsync( + RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null) + { + try + { + return await RemoveAllAsync(query, options); + } + catch (RepositoryException ex) when (IsIndexNotFound(ex.InnerException as TransportException)) + { + return 0; + } + catch (TransportException ex) when (IsIndexNotFound(ex)) + { + return 0; + } + } + + private static bool IsIndexNotFound(TransportException? ex) + { + if (ex?.ApiCallDetails?.HttpStatusCode != 404) + return false; + + return ex.ApiCallDetails.ProductError is ElasticsearchServerError serverError + ? IsIndexNotFound(serverError) + : ex.DebugInformation.Contains("index_not_found_exception", StringComparison.Ordinal); + } + + private static bool IsIndexNotFound(ElasticsearchServerError serverError) + { + if (serverError.Status != 404 || serverError.Error is null) + return false; + + return String.Equals(serverError.Error.Type, "index_not_found_exception", StringComparison.Ordinal) + || serverError.Error.RootCause?.Any(IsIndexNotFound) == true; + } + + private static bool IsIndexNotFound(Elastic.Transport.Products.Elasticsearch.ErrorCause? cause) + { + return cause is not null + && (String.Equals(cause.Type, "index_not_found_exception", StringComparison.Ordinal) + || cause.CausedBy is not null && IsIndexNotFound(cause.CausedBy)); + } + + /// + /// Reassigns all events from the source stacks to the target stack using a parameterized + /// Painless script (no string interpolation) to prevent script injection. + /// + public Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId) + { + ArgumentNullException.ThrowIfNull(sourceStackIds); + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + + // Materialize to avoid multiple enumeration and guard against empty; an empty + // .Stack() filter would match ALL events and reassign them to the target stack. + var sourceIds = sourceStackIds.ToList(); + if (sourceIds.Count == 0) + return Task.FromResult(0L); + + return PatchAllAsync( + q => q.Stack(sourceIds), + new ScriptPatch("ctx._source.stack_id = params.targetStackId") + { + Params = new Dictionary { ["targetStackId"] = targetStackId } + }); + } + + public Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("stack_id", e => e.StackId, batchSize, afterValue, cancellationToken); + } + + public Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("project_id", e => e.ProjectId, batchSize, afterValue, cancellationToken); + } + + public Task GetDistinctOrganizationIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("organization_id", e => e.OrganizationId, batchSize, afterValue, cancellationToken); + } + + /// + /// Uses a composite aggregation to paginate through all distinct values of a field. + /// Composite aggregations are preferred over terms aggregations for high-cardinality fields + /// because terms aggregations can silently miss values when the unique count exceeds the + /// configured size parameter. Composite aggregations guarantee correct iteration via an + /// after_key cursor, at the cost of requiring sequential page fetches. + /// + private async Task GetDistinctFieldValuesAsync( + string fieldName, + Expression> fieldExpression, + int batchSize, + string? afterValue, + CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(batchSize); + + string aggregationName = $"composite_{fieldName}"; + var sources = new List> + { + new(fieldName, new CompositeAggregationSource + { + Terms = new CompositeTermsAggregation { Field = fieldExpression } + }) + }; + + var search = await _configuration.Client.SearchAsync(s => + { + s.Indices($"{_configuration.Events.VersionedName}-*") + .Size(0) + .AddAggregation(aggregationName, a => a.Composite(c => + { + c.Size(batchSize) + .Sources(sources); + + if (!String.IsNullOrEmpty(afterValue)) + c.After(new Dictionary { [fieldName] = afterValue }); + })); + }, cancellationToken); + + if (!search.IsValidResponse) + { + if (search.ElasticsearchServerError is not null && IsIndexNotFound(search.ElasticsearchServerError)) + return new DistinctValuePage([], null); + + throw new InvalidOperationException($"Error retrieving distinct event values for '{fieldName}': {search.DebugInformation}", search.ApiCallDetails.OriginalException); + } + + var composite = search.Aggregations?.GetComposite(aggregationName); + var values = composite?.Buckets is { Count: > 0 } + ? composite.Buckets.Select(bucket => bucket.Key[fieldName].ToString()!).ToArray() + : []; + string? nextValue = composite?.AfterKey is not null && composite.AfterKey.TryGetValue(fieldName, out var next) + ? next.ToString() + : null; + + return new DistinctValuePage(values, nextValue); } } diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index c7c2272cbc..2c0e4ea69c 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -13,8 +13,16 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); + Task RemoveAllByProjectIdsAsync(string[] projectIds); + Task RemoveAllByOrganizationIdsAsync(string[] organizationIds); + Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId); + Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); + Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); + Task GetDistinctOrganizationIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); } +public sealed record DistinctValuePage(IReadOnlyCollection Values, string? NextValue); + public static class EventRepositoryExtensions { public static async Task GetPreviousAndNextEventIdsAsync(this IEventRepository repository, string id, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null) diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs index 13199d28c6..f2ec72337a 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs @@ -15,4 +15,5 @@ public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject> GetStacksForCleanupAsync(string organizationId, DateTime cutoff); Task> GetSoftDeleted(); Task SoftDeleteByProjectIdAsync(string organizationId, string projectId); + Task> GetDuplicateSignaturesAsync(int maxResults = 10000); } diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index 29b0c3407e..adc0524d72 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -187,6 +187,22 @@ public Task SoftDeleteByProjectIdAsync(string organizationId, string proje ); } + public async Task> GetDuplicateSignaturesAsync(int maxResults = 10000) + { + // ImmediateConsistency forces a segment refresh before the aggregation so that + // any stacks soft-deleted in a previous batch are excluded here. Cost: one refresh + // per batch (not per item), equivalent to the original explicit index refresh. + var result = await CountAsync( + q => q.AggregationsExpression($"terms:(duplicate_signature~{maxResults} @min:2)"), + o => o.ImmediateConsistency()); + + var buckets = result.Aggregations.Terms("terms_duplicate_signature")?.Buckets; + if (buckets is not { Count: > 0 }) + return []; + + return buckets.Select(b => b.Key).ToArray(); + } + protected override async Task AddDocumentsToCacheAsync(ICollection> findHits, ICommandOptions options, bool isDirtyRead) { await base.AddDocumentsToCacheAsync(findHits, options, isDirtyRead); diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index 0ab2d63e52..143d15f50a 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -17,6 +17,7 @@ + diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs new file mode 100644 index 0000000000..c75a98ecf5 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs @@ -0,0 +1,160 @@ +using Exceptionless.Core.Models; +using Exceptionless.DateTimeExtensions; +using Exceptionless.Tests.Utility; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public partial class CleanupDataJobTests +{ + [Fact] + public async Task CleanupSoftDeletedOrganizations_WithMultiplePages_RemovesAllData() + { + // Arrange + var organizations = new List(); + for (int i = 0; i < 12; i++) + { + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + organization.IsDeleted = true; + organizations.Add(organization); + } + + await _organizationRepository.AddAsync(organizations, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organizations[0].Id), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organizations[0].Id, projectId: project.Id), + o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvents(10, organizations[0].Id, project.Id, stack.Id), + o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + foreach (var organization in organizations) + Assert.Null(await _organizationRepository.GetByIdAsync(organization.Id, o => o.IncludeSoftDeletes())); + + Assert.Null(await _projectRepository.GetByIdAsync(project.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); + Assert.Equal(0, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrgRetention() + { + // Arrange + // Retention enforcement uses the next plan above the organization's retention: + // FreePlan (3d) becomes 30d and SmallPlan (30d) becomes 90d. + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization1, _plans.SmallPlan); + organization1.StripeCustomerId = "cust_test1"; + organization1.CardLast4 = "4242"; + organization1.SubscribeDate = DateTime.UtcNow; + organization1.BillingChangedByUserId = TestConstants.UserId; + + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization2, _plans.FreePlan); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(generateId: true, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var recentStart = DateTimeOffset.UtcNow.AddDays(-2); + var recentEnd = DateTimeOffset.UtcNow.AddDays(-1); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization1.Id, project1.Id, stack1.Id, startDate: recentStart, endDate: recentEnd), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization2.Id, project2.Id, stack2.Id, startDate: recentStart, endDate: recentEnd), o => o.ImmediateConsistency()); + + var olderStart = DateTimeOffset.UtcNow.AddDays(-37); + var olderEnd = DateTimeOffset.UtcNow.AddDays(-33); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization1.Id, project1.Id, stack1.Id, startDate: olderStart, endDate: olderEnd), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization2.Id, project2.Id, stack2.Id, startDate: olderStart, endDate: olderEnd), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(20, await _eventRepository.CountAsync(q => q.FilterExpression($"organization:{organization1.Id}"))); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.FilterExpression($"organization:{organization2.Id}"))); + } + + [Fact] + public async Task CleanupSoftDeletedStacks_WithMultiplePages_RemovesAllStacks() + { + // Arrange + var organization = await _organizationRepository.AddAsync( + _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true), + o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organization.Id), + o => o.ImmediateConsistency()); + + var stacks = new List(); + for (int i = 0; i < 600; i++) + { + var stack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + stack.IsDeleted = true; + stacks.Add(stack); + } + + await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); + + var validStack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id), + o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvents(5, organization.Id, project.Id, validStack.Id), + o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(1, await _stackRepository.CountAsync(o => o.IncludeSoftDeletes())); + Assert.Equal(5, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task EnforceRetention_WithEventsOutsideRetention_DeletesOnlyExpiredEvents() + { + // Arrange + // FreePlan's 3-day retention is enforced at the next plan threshold (30 days). + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization, _plans.FreePlan); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organization.Id), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id), + o => o.ImmediateConsistency()); + + var outsideRetentionStart = DateTimeOffset.UtcNow.SubtractDays(37); + var outsideRetentionEnd = DateTimeOffset.UtcNow.SubtractDays(33); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, stack.Id, startDate: outsideRetentionStart, endDate: outsideRetentionEnd), o => o.ImmediateConsistency()); + + var insideRetentionStart = DateTimeOffset.UtcNow.SubtractDays(2); + var insideRetentionEnd = DateTimeOffset.UtcNow.SubtractDays(1); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, stack.Id, startDate: insideRetentionStart, endDate: insideRetentionEnd), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(10, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } +} diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index c7300641a2..786391f8bd 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -1,8 +1,8 @@ using Exceptionless.Core; using Exceptionless.Core.Authorization; using Exceptionless.Core.Billing; -using Exceptionless.Core.Jobs; using Exceptionless.Core.Extensions; +using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; @@ -17,7 +17,7 @@ namespace Exceptionless.Tests.Jobs; -public class CleanupDataJobTests : IntegrationTestsBase +public partial class CleanupDataJobTests : IntegrationTestsBase { private readonly CleanupDataJob _job; private readonly UsageService _usageService; @@ -61,7 +61,7 @@ public CleanupDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) } [Fact] - public async Task CanCleanupSuspendedTokens() + public async Task RunAsync_SuspendedOrganization_SuspendsRelatedTokens() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); organization.IsSuspended = true; @@ -136,7 +136,7 @@ OAuthToken CreateOAuthToken(DateTime updatedUtc, bool isDisabled, string? refres } [Fact] - public async Task CanCleanupSoftDeletedOrganization() + public async Task RunAsync_SoftDeletedOrganization_RemovesAllRelatedData() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); organization.IsDeleted = true; @@ -238,7 +238,7 @@ public async Task CleanupSyntheticUsersAsync_FreshMemberAndSimilarUsers_KeepsUse } [Fact] - public async Task CanCleanupSoftDeletedProject() + public async Task RunAsync_SoftDeletedProject_RemovesProjectAndEvents() { var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); @@ -258,7 +258,7 @@ public async Task CanCleanupSoftDeletedProject() } [Fact] - public async Task CanCleanupSoftDeletedStack() + public async Task RunAsync_SoftDeletedStack_RemovesStackAndEvents() { var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); @@ -278,7 +278,7 @@ public async Task CanCleanupSoftDeletedStack() } [Fact] - public async Task CanCleanupEventsOutsideOfRetentionPeriod() + public async Task RunAsync_EventsOutsideRetentionPeriod_RemovesExpiredEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); _billingManager.ApplyBillingPlan(organization, _plans.FreePlan); @@ -300,7 +300,7 @@ public async Task CanCleanupEventsOutsideOfRetentionPeriod() } [Fact] - public async Task CanDeleteOrphanedEventsByStack() + public async Task DeleteOrphanedEventsByStack_WithLargeDataset_DeletesAllOrphanedEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -898,4 +898,5 @@ private static OAuthToken CreateUserOAuthToken(DateTime utcNow, string userId) UpdatedUtc = utcNow }; } + } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index af3f405b16..4aafe47c1d 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -6,6 +6,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; +using Foundatio.Utility; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -42,7 +43,7 @@ public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory f [Fact] public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents() { - // Arrange - Two tenants, each with valid stacks and events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -62,7 +63,7 @@ public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents // Act await _job.RunAsync(TestCancellationToken); - // Assert - All events should remain (no orphans) + // Assert var totalCount = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(200, totalCount); } @@ -70,7 +71,7 @@ public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents [Fact] public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() { - // Arrange - Tenant 1 has valid events; Tenant 2 has orphaned events (stack doesn't exist) + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -83,15 +84,10 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); - // Valid events for both tenants var validEvents1 = _eventData.GenerateEvents(50, organization1.Id, project1.Id, stack1.Id).ToList(); var validEvents2 = _eventData.GenerateEvents(50, organization2.Id, project2.Id, stack2.Id).ToList(); - - // Orphaned events (stack IDs that don't exist) in both tenants - string fakeStackId1 = ObjectId.GenerateNewId().ToString(); - string fakeStackId2 = ObjectId.GenerateNewId().ToString(); - var orphanedEvents1 = _eventData.GenerateEvents(30, organization1.Id, project1.Id, fakeStackId1).ToList(); - var orphanedEvents2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, fakeStackId2).ToList(); + var orphanedEvents1 = _eventData.GenerateEvents(30, organization1.Id, project1.Id, ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedEvents2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, ObjectId.GenerateNewId().ToString()).ToList(); await _eventRepository.AddAsync(validEvents1.Concat(validEvents2).Concat(orphanedEvents1).Concat(orphanedEvents2), o => o.ImmediateConsistency()); @@ -101,7 +97,7 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } @@ -109,7 +105,7 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele [Fact] public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvents() { - // Arrange - Large volume across two tenants: 5000 valid + 10000 orphaned + // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -119,10 +115,8 @@ public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvent var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization.Id, projectId: project.Id); await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); - // 5000 valid events for existing stack await _eventRepository.AddAsync(_eventData.GenerateEvents(5000, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); - // 10000 orphaned events with many different fake stack IDs var orphanedEvents = _eventData.GenerateEvents(10000, organization.Id, project.Id).ToList(); orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); @@ -133,15 +127,42 @@ public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvent // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only the 5000 valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(5000, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByStack_WithManyUniqueOrphanedStacks_DeletesAllOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var orphanedEvents = _eventData.GenerateEvents(200, organization.Id, project.Id).ToList(); + var uniqueStackIds = Enumerable.Range(0, 200).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + for (int i = 0; i < orphanedEvents.Count; i++) + orphanedEvents[i].StackId = uniqueStackIds[i]; + + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(250, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() { - // Arrange - Multiple valid stacks in two organizations, no orphans + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -150,7 +171,6 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); - // Multiple stacks per project var stacks = new List(); for (int i = 0; i < 10; i++) { @@ -159,7 +179,6 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() } await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); - // Events across all stacks var events = new List(); foreach (var stack in stacks) events.AddRange(_eventData.GenerateEvents(10, stack.OrganizationId, stack.ProjectId, stack.Id)); @@ -168,7 +187,7 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() // Act await _job.RunAsync(TestCancellationToken); - // Assert - All 200 events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(200, totalAfter); } @@ -176,7 +195,7 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() [Fact] public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_OtherTenantUnaffected() { - // Arrange - Tenant 1 has all orphaned events (will be deleted); Tenant 2 has all valid events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -185,15 +204,11 @@ public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_Othe var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); - // Tenant 2 has a valid stack var validStack = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // Tenant 1 events are all orphaned (fake stack IDs) var orphanedEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id).ToList(); orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); - - // Tenant 2 events are all valid var validEvents = _eventData.GenerateEvents(100, organization2.Id, project2.Id, validStack.Id).ToList(); await _eventRepository.AddAsync(orphanedEvents.Concat(validEvents), o => o.ImmediateConsistency()); @@ -201,11 +216,36 @@ public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_Othe // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only tenant 2's events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByStack_WithSoftDeletedStack_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + softDeletedStack.IsDeleted = true; + softDeletedStack = await _stackRepository.AddAsync(softDeletedStack, o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, softDeletedStack.Id), o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteEvents() { @@ -236,7 +276,7 @@ public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteE [Fact] public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEventsForMissingProject() { - // Arrange - Events reference a project that doesn't exist + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -247,10 +287,7 @@ public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEvent var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: validProject.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // Valid events for existing project var validEvents = _eventData.GenerateEvents(75, organization1.Id, validProject.Id, validStack.Id).ToList(); - - // Orphaned events referencing a non-existent project in organization 2 string fakeProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); var orphanedEvents = _eventData.GenerateEvents(50, organization2.Id, fakeProjectId, fakeStackId).ToList(); @@ -260,30 +297,50 @@ public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEvent // Act await _job.RunAsync(TestCancellationToken); - // Assert - Orphaned events deleted, valid events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(75, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByProject_WithMissingProjects_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + string fakeProjectId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(100, organization.Id, fakeProjectId).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependent() { - // Arrange - Tenant 1 has valid project, Tenant 2 has orphaned project + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); - // Only Tenant 1 has a real project var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); - // Tenant 1 valid events var validEvents = _eventData.GenerateEvents(60, organization1.Id, project1.Id, stack1.Id).ToList(); - - // Tenant 2 orphaned events (project doesn't exist) string nonExistentProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); var orphanedEvents = _eventData.GenerateEvents(40, organization2.Id, nonExistentProjectId, fakeStackId).ToList(); @@ -298,6 +355,33 @@ public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependen Assert.Equal(60, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByProject_WithSoftDeletedProject_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedProject = _projectData.GenerateProject(generateId: true, organizationId: organization.Id); + softDeletedProject.IsDeleted = true; + softDeletedProject = await _projectRepository.AddAsync(softDeletedProject, o => o.ImmediateConsistency()); + var orphanedEvents = _eventData.GenerateEvents(100, organization.Id, softDeletedProject.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_DoesNotDeleteEvents() { @@ -321,7 +405,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_Does // Act await _job.RunAsync(TestCancellationToken); - // Assert - All events preserved (both organizations exist) + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(160, totalAfter); } @@ -329,7 +413,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_Does [Fact] public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_DeletesEventsForMissingOrganization() { - // Arrange - Valid organization1 + events referencing non-existent organization + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -339,10 +423,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_De var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project.Id); await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); - // Valid events var validEvents = _eventData.GenerateEvents(100, organization1.Id, project.Id, stack.Id).ToList(); - - // Orphaned events referencing a non-existent organization string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); string fakeProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); @@ -353,15 +434,39 @@ public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_De // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only valid events survive + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithMissingOrganizations_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(100, fakeOrganizationId, project.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDeletesOrphanedTenantEvents() { - // Arrange - Organization 1 exists, Organization 2 does NOT exist (never created) but has events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -371,10 +476,7 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); - // Tenant 1 valid events var validEvents = _eventData.GenerateEvents(120, organization1.Id, project1.Id, stack1.Id).ToList(); - - // Tenant 2 events (organization doesn't exist, simulates post-hard-delete orphans) string ghostOrganizationId = ObjectId.GenerateNewId().ToString(); string ghostProjectId = ObjectId.GenerateNewId().ToString(); string ghostStackId = ObjectId.GenerateNewId().ToString(); @@ -390,17 +492,43 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe Assert.Equal(120, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithSoftDeletedOrganization_DeletesOrphanedEvents() + { + // Arrange + var validOrganization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(validOrganization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, validOrganization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedOrganization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + softDeletedOrganization.IsDeleted = true; + softDeletedOrganization = await _organizationRepository.AddAsync(softDeletedOrganization, o => o.ImmediateConsistency()); + var orphanedEvents = _eventData.GenerateEvents(100, softDeletedOrganization.Id, project.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly() { - // Arrange - Two stacks in the same project with the same signature (duplicate) + // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); - string signatureHash = "abc123def456"; + const string signatureHash = "abc123def456"; var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); stack1.CreatedUtc = DateTime.UtcNow.AddDays(-10); stack1.TotalOccurrences = 5; @@ -409,7 +537,6 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly stack2.TotalOccurrences = 10; await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); - // Events on both stacks var events1 = _eventData.GenerateEvents(3, organization.Id, project.Id, stack1.Id).ToList(); var events2 = _eventData.GenerateEvents(7, organization.Id, project.Id, stack2.Id).ToList(); await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); @@ -417,7 +544,7 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly // Act await _job.RunAsync(TestCancellationToken); - // Assert - One stack should be deleted, all events should point to the surviving stack + // Assert await RefreshDataAsync(); var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); var activeStacks = allStacks.Documents.Where(s => !s.IsDeleted).ToList(); @@ -425,16 +552,53 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly Assert.Single(activeStacks); Assert.Single(deletedStacks); - // All events should now reference the surviving stack var allEvents = await _eventRepository.GetAllAsync(); Assert.Equal(10, allEvents.Total); Assert.All(allEvents.Documents, e => Assert.Equal(activeStacks[0].Id, e.StackId)); } + [Fact] + public async Task FixDuplicateStacks_WithDuplicateSignatures_MergesIntoMostPopularStack() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + originalStack.TotalOccurrences = 100; + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + duplicateStack.TotalOccurrences = 10; + + originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency()); + duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, originalStack.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, duplicateStack.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedOriginal = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); + var updatedDuplicate = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes()); + + Assert.NotNull(updatedOriginal); + Assert.NotNull(updatedDuplicate); + Assert.False(updatedOriginal.IsDeleted); + Assert.True(updatedDuplicate.IsDeleted); + Assert.Equal(110, updatedOriginal.TotalOccurrences); + + Assert.Equal(110, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + Assert.Equal(110, await _eventRepository.CountAsync(q => q.Stack(originalStack.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id))); + } + [Fact] public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() { - // Arrange - Two stacks with different signatures across two tenants + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -454,7 +618,7 @@ public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() // Act await _job.RunAsync(TestCancellationToken); - // Assert - Nothing deleted + // Assert var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); Assert.Equal(2, allStacks.Total); Assert.All(allStacks.Documents, s => Assert.False(s.IsDeleted)); @@ -463,10 +627,41 @@ public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() Assert.Equal(40, totalEvents); } + [Fact] + public async Task FixDuplicateStacks_WithNoEvents_KeepsOldestStack() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + originalStack.CreatedUtc = DateTime.UtcNow.AddMinutes(-10); + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + duplicateStack.CreatedUtc = originalStack.CreatedUtc.AddMinutes(1); + + originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency()); + duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedOriginal = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); + var updatedDuplicate = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes()); + + Assert.NotNull(updatedOriginal); + Assert.NotNull(updatedDuplicate); + Assert.False(updatedOriginal.IsDeleted); + Assert.True(updatedDuplicate.IsDeleted); + } + [Fact] public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() { - // Arrange - Complex scenario: valid data, orphaned by stack, orphaned by project, orphaned by organization + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -476,23 +671,10 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // 100 valid events var validEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id, validStack.Id).ToList(); - - // 25 orphaned by stack (stack doesn't exist) - string fakeStack = ObjectId.GenerateNewId().ToString(); - var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, fakeStack).ToList(); - - // 25 orphaned by project (project doesn't exist) - string fakeProject = ObjectId.GenerateNewId().ToString(); - string fakeStack2 = ObjectId.GenerateNewId().ToString(); - var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, fakeProject, fakeStack2).ToList(); - - // 25 orphaned by organization (organization doesn't exist) - string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); - string fakeProject2 = ObjectId.GenerateNewId().ToString(); - string fakeStack3 = ObjectId.GenerateNewId().ToString(); - var orphanedByOrganization = _eventData.GenerateEvents(25, fakeOrganizationId, fakeProject2, fakeStack3).ToList(); + var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedByOrganization = _eventData.GenerateEvents(25, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedByStack).Concat(orphanedByProject).Concat(orphanedByOrganization), o => o.ImmediateConsistency()); @@ -502,7 +684,7 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only 100 valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } @@ -510,7 +692,7 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() [Fact] public async Task RunAsync_NoOrphans_PreservesEverything() { - // Arrange - Two complete tenants, no orphans anywhere + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -530,7 +712,7 @@ public async Task RunAsync_NoOrphans_PreservesEverything() // Act await _job.RunAsync(TestCancellationToken); - // Assert - All 400 events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(400, totalAfter); } @@ -538,11 +720,12 @@ public async Task RunAsync_NoOrphans_PreservesEverything() [Fact] public async Task RunAsync_EmptyDatabase_CompletesWithoutError() { - // Arrange - nothing + // Arrange - // Act & Assert - should not throw + // Act await _job.RunAsync(TestCancellationToken); + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(0, totalAfter); } diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 95c5d2b628..3dffd0aa6a 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -275,4 +275,186 @@ private async Task CreateDataAsync() _ids.Add(Tuple.Create(ev.Id, date)); } } + + [Fact] + public async Task GetDistinctStackIds_WithMultipleStacks_ReturnsAllUniqueIds() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(5, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack2.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctStackIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var stackIds = page.Values; + + // Assert + Assert.Equal(stackIds.Count, stackIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(stack1.Id, stackIds); + Assert.Contains(stack2.Id, stackIds); + } + + [Fact] + public async Task GetDistinctStackIds_WithPagination_ReturnsAllIds() + { + // Arrange + var stacks = new List(); + for (int i = 0; i < 5; i++) + stacks.Add(await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency())); + + foreach (var stack in stacks) + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act - page through with batch size of 2 + var allIds = new List(); + string? nextValue = null; + do + { + var page = await _repository.GetDistinctStackIdsAsync(2, nextValue, TestContext.Current.CancellationToken); + allIds.AddRange(page.Values); + nextValue = page.NextValue; + } while (!String.IsNullOrEmpty(nextValue)); + + // Assert + Assert.Equal(allIds.Count, allIds.Distinct(StringComparer.Ordinal).Count()); + foreach (var stack in stacks) + Assert.Contains(stack.Id, allIds); + } + + [Fact] + public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(5, TestConstants.OrganizationId, TestConstants.ProjectId, stack2.Id), o => o.ImmediateConsistency()); + + // Act + long affected = await _repository.ReassignStackAsync([stack1.Id], stack2.Id); + + // Assert + Assert.Equal(10, affected); + + await RefreshDataAsync(); + + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack1.Id))); + Assert.Equal(15, await _repository.CountAsync(q => q.Stack(stack2.Id))); + } + + [Fact] + public async Task RemoveAllByProjectIds_WithMatchingEvents_RemovesAll() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByProjectIdsAsync([TestConstants.ProjectId]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task RemoveAllByOrganizationIds_WithMatchingEvents_RemovesAll() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByOrganizationIdsAsync([TestConstants.OrganizationId]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task RemoveAllByStackIds_WithMatchingEvents_RemovesAll() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByStackIdsAsync([stack.Id]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task ReassignStack_WithEmptySourceIds_ReturnsZeroWithoutModification() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + + // Act - empty source list must be a no-op; an unchecked empty .Stack() filter would patch ALL events + long affected = await _repository.ReassignStackAsync([], stack2.Id); + + // Assert + Assert.Equal(0, affected); + + await RefreshDataAsync(); + Assert.Equal(10, await _repository.CountAsync(q => q.Stack(stack1.Id))); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack2.Id))); + } + + [Fact] + public async Task GetDistinctProjectIds_WithMultipleProjects_ReturnsAllUniqueIds() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string project2Id = ObjectId.GenerateNewId().ToString(); + string projectWithoutEventsId = ObjectId.GenerateNewId().ToString(); + await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: projectWithoutEventsId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, project2Id, stack.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctProjectIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var projectIds = page.Values; + + // Assert + Assert.Equal(projectIds.Count, projectIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(TestConstants.ProjectId, projectIds); + Assert.Contains(project2Id, projectIds); + Assert.DoesNotContain(projectWithoutEventsId, projectIds); + } + + [Fact] + public async Task GetDistinctOrganizationIds_WithMultipleOrganizations_ReturnsAllUniqueIds() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string org2Id = ObjectId.GenerateNewId().ToString(); + + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, org2Id, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctOrganizationIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var orgIds = page.Values; + + // Assert + Assert.Equal(orgIds.Count, orgIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(TestConstants.OrganizationId, orgIds); + Assert.Contains(org2Id, orgIds); + } } diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 54b4093187..76270de5c3 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -8,7 +8,9 @@ using Foundatio.Caching; using Foundatio.Repositories; using Foundatio.Repositories.Options; +using Foundatio.Repositories.Utility; using Foundatio.Serializer; +using Foundatio.Utility; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -279,4 +281,58 @@ await _repository.AddAsync( Assert.NotNull(stacks.Documents.SingleOrDefault(s => String.Equals(s.Id, TestConstants.StackId))); Assert.NotNull(stacks.Documents.SingleOrDefault(s => String.Equals(s.Id, TestConstants.StackId2))); } + + [Fact] + public async Task GetDuplicateSignatures_WithDuplicates_ReturnsSignatures() + { + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:dup_sig_test"; + + var stack2 = stack1.DeepClone(); + stack2.Id = ObjectId.GenerateNewId().ToString(); + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + Assert.Contains($"{uniqueProjectId}:dup_sig_test", duplicates); + } + + [Fact] + public async Task GetDuplicateSignatures_WithNoDuplicates_ReturnsEmpty() + { + // Use a unique project ID to avoid interference from pre-existing sample data + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:unique_sig_1"; + + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack2.DuplicateSignature = $"{uniqueProjectId}:unique_sig_2"; + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + // Should not contain our unique signatures since they each appear only once + Assert.DoesNotContain($"{uniqueProjectId}:unique_sig_1", duplicates); + Assert.DoesNotContain($"{uniqueProjectId}:unique_sig_2", duplicates); + } + + [Fact] + public async Task GetDuplicateSignatures_WithSoftDeletedStacks_ExcludesThem() + { + // Use a unique project ID to avoid interference from pre-existing sample data + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:softdelete_sig"; + + var stack2 = stack1.DeepClone(); + stack2.Id = ObjectId.GenerateNewId().ToString(); + stack2.IsDeleted = true; + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + // The soft-deleted stack should be excluded, leaving only 1 stack with this signature + Assert.DoesNotContain($"{uniqueProjectId}:softdelete_sig", duplicates); + } } From f6fcc4b9524a718fa9f45e1e1d38adca80645b7a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 12 Jul 2026 14:46:56 -0500 Subject: [PATCH 2/9] fix: make cleanup jobs lossless under concurrency --- .../Jobs/CleanupOrphanedDataJob.cs | 194 ++++++++-- src/Exceptionless.Core/Jobs/StackStatusJob.cs | 8 +- src/Exceptionless.Core/Models/Stack.cs | 33 +- .../Pipeline/010_AssignToStackAction.cs | 6 +- .../Configuration/Indexes/StackIndex.cs | 2 + .../Repositories/EventRepository.cs | 88 ++++- .../Interfaces/IEventRepository.cs | 7 +- .../Interfaces/IStackRepository.cs | 7 + .../Repositories/StackRepository.cs | 357 +++++++++++++++++- ...IgnoreForExternalSerializationAttribute.cs | 7 + .../JsonSerializerOptionsExtensions.cs | 13 + .../Exceptionless.Web.csproj | 1 - .../Jobs/CleanupDataJobPaginationTests.cs | 2 +- .../Jobs/CleanupOrphanedDataJobTests.cs | 168 +++++++++ .../Repositories/EventRepositoryTests.cs | 37 +- .../Repositories/StackRepositoryTests.cs | 298 +++++++++++++++ .../Utility/PublicApiCompatibilityTests.cs | 21 ++ 17 files changed, 1165 insertions(+), 84 deletions(-) create mode 100644 src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 5e6e4bc257..a8785970ef 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -4,6 +4,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Repositories; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; using Foundatio.Resilience; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -88,12 +89,35 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) if (missingStackIds.Length == 0) continue; - long deletedCount = await _eventRepository.RemoveAllByStackIdsAsync(missingStackIds); + // Redirect tombstones are intentionally retained so events from an in-flight ingestion + // context can never be mistaken for orphaned data. Move those late events to the + // canonical stack and refresh its metadata before deleting only truly missing stacks. + var redirectedStacks = await _stackRepository.GetByIdsAsync( + missingStackIds, + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + var redirectedStackIds = redirectedStacks + .Where(s => !String.IsNullOrEmpty(s.RedirectToStackId)) + .Select(s => s.Id) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var redirectedStackDocuments = redirectedStacks + .Where(s => redirectedStackIds.Contains(s.Id)) + .ToList(); + + if (redirectedStackDocuments.Count > 0) + await ReconcileRedirectedStacksAsync(redirectedStackDocuments, context); + + string[] orphanedStackIds = missingStackIds.Where(id => !redirectedStackIds.Contains(id)).ToArray(); + if (orphanedStackIds.Length == 0) + continue; + + long deletedCount = await _eventRepository.RemoveAllByStackIdsAsync(orphanedStackIds, o => o.Notifications(false)); totalOrphanedEvents += deletedCount; - _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingStackCount} missing stacks out of {StackIdCount} checked", deletedCount, missingStackIds.Length, stackIds.Count); + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingStackCount} missing stacks out of {StackIdCount} checked", deletedCount, orphanedStackIds.Length, stackIds.Count); } + await ReconcileDirtyRedirectedStacksAsync(context); + _logger.LogInformation("Completed orphaned events cleanup by stack: deleted {TotalOrphanedEvents} events, checked {TotalStackIds} stacks", totalOrphanedEvents, totalStackIds); } @@ -125,7 +149,7 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) if (missingProjectIds.Length == 0) continue; - long deletedCount = await _eventRepository.RemoveAllByProjectIdsAsync(missingProjectIds); + long deletedCount = await _eventRepository.RemoveAllByProjectIdsAsync(missingProjectIds, o => o.Notifications(false)); totalOrphanedEvents += deletedCount; _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingProjectCount} missing projects out of {ProjectIdCount} checked", deletedCount, missingProjectIds.Length, projectIds.Count); @@ -162,7 +186,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) if (missingOrganizationIds.Length == 0) continue; - long deletedCount = await _eventRepository.RemoveAllByOrganizationIdsAsync(missingOrganizationIds); + long deletedCount = await _eventRepository.RemoveAllByOrganizationIdsAsync(missingOrganizationIds, o => o.Notifications(false)); totalOrphanedEvents += deletedCount; _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingOrganizationCount} missing organizations out of {OrganizationIdCount} checked", deletedCount, missingOrganizationIds.Length, organizationIds.Count); @@ -222,6 +246,13 @@ public async Task FixDuplicateStacks(JobContext context) continue; } + var targetCandidates = stacks.Documents.Where(s => String.IsNullOrEmpty(s.RedirectToStackId)).ToList(); + if (targetCandidates.Count == 0) + { + _logger.LogError("Did not find a canonical stack for signature {SignatureHash} and project {ProjectId}", signature, projectId); + continue; + } + var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); @@ -229,44 +260,44 @@ public async Task FixDuplicateStacks(JobContext context) bool shouldUpdateEvents = eventCountBuckets.Count > 1; // Default to using the oldest stack. - var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First(); - var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList(); + var targetStack = targetCandidates.OrderBy(s => s.CreatedUtc).First(); + var duplicateStacks = stacks.Documents.Where(s => s.Id != targetStack.Id).OrderBy(s => s.CreatedUtc).ToList(); // Use the stack that has the most events on it so we can reduce the number of updates. - if (eventCountBuckets.Count > 0) + var targetCandidateIds = targetCandidates.Select(s => s.Id).ToHashSet(StringComparer.Ordinal); + var targetBuckets = eventCountBuckets.Where(b => targetCandidateIds.Contains(b.Key)).ToList(); + if (targetBuckets.Count > 0) { - string targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key; - targetStack = stacks.Documents.Single(d => d.Id == targetStackId); - duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList(); + string targetStackId = targetBuckets.OrderByDescending(b => b.Total).First().Key; + targetStack = targetCandidates.Single(d => d.Id == targetStackId); + duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).OrderBy(d => d.CreatedUtc).ToList(); } - targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc); - targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open; - targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence); - targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc); - targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed); - targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences); - targetStack.Tags.UnionWith(duplicateStacks.SelectMany(d => d.Tags)); - targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList(); - targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical); + // Publish durable redirects before moving events. New ingestion resolves these + // immediately, and any already in-flight writes are preserved by the orphan pass. + foreach (var duplicateStack in duplicateStacks) + await _stackRepository.SetDuplicateStackRedirectAsync(duplicateStack, targetStack.Id); - duplicateStacks.ForEach(s => s.IsDeleted = true); + // Always run strict verification, even when open-index counts show no source + // events. Closed or unavailable daily indices must block stack deletion. + long affectedRecords = await AwaitWithLockRenewalAsync( + _eventRepository.ReassignStackAsync( + duplicateStacks.Select(s => s.Id), targetStack.Id, context.CancellationToken), + context); - if (shouldUpdateEvents) - { - // Reassign events before soft-deleting duplicates: if event reassignment - // fails, the duplicate stacks remain visible and no data is lost. - long affectedRecords = await _eventRepository.ReassignStackAsync( - duplicateStacks.Select(s => s.Id), targetStack.Id); + _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); + totalUpdatedEventCount += affectedRecords; + + context.CancellationToken.ThrowIfCancellationRequested(); - _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); - totalUpdatedEventCount += affectedRecords; + // Apply each source's metadata to the target using a durable occurrence ledger, + // then hide the redirected source. This ordering is retry-safe when any write fails. + foreach (var duplicateStack in duplicateStacks) + { + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, duplicateStack); + await _stackRepository.SetDuplicateStackRedirectAsync(duplicateStack, targetStack.Id, isDeleted: true); } - // Soft-delete duplicates and save after events are safely migrated. - // No per-item ImmediateConsistency needed: GetDuplicateSignaturesAsync - // forces a refresh before each batch aggregation call. - await _stackRepository.SaveAsync([.. duplicateStacks, targetStack]); processed++; batchProcessed++; @@ -305,12 +336,111 @@ public async Task FixDuplicateStacks(JobContext context) _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount} UpdatedEvents={UpdatedEventCount}", processed, total, error, totalUpdatedEventCount); } + private async Task ReconcileDirtyRedirectedStacksAsync(JobContext context) + { + var redirectedStacks = await _stackRepository.GetRedirectedStacksNeedingReconciliationAsync(); + while (redirectedStacks.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) + { + await ReconcileRedirectedStacksAsync(redirectedStacks.Documents, context); + + if (!await redirectedStacks.NextPageAsync()) + break; + } + } + + private async Task ReconcileRedirectedStacksAsync(IReadOnlyCollection redirectedStacks, JobContext context) + { + var stacksByTarget = new Dictionary Sources)>(StringComparer.Ordinal); + foreach (var sourceStack in redirectedStacks) + { + context.CancellationToken.ThrowIfCancellationRequested(); + + try + { + var targetStack = await _stackRepository.GetCanonicalStackAsync(sourceStack.RedirectToStackId!); + if (targetStack is null) + { + _logger.LogWarning( + "Preserving redirected stack {SourceStackId} because target stack {TargetStackId} is unavailable", + sourceStack.Id, + sourceStack.RedirectToStackId); + continue; + } + + if (!stacksByTarget.TryGetValue(targetStack.Id, out var group)) + { + group = (targetStack, []); + stacksByTarget[targetStack.Id] = group; + } + + group.Sources.Add(sourceStack); + } + catch (DocumentException ex) + { + _logger.LogError( + ex, + "Unable to resolve redirected stack {SourceStackId} to target {TargetStackId}", + sourceStack.Id, + sourceStack.RedirectToStackId); + } + } + + foreach (var (_, group) in stacksByTarget) + { + // Merge metadata first. If event reassignment fails, the contribution ledger makes + // this safe to retry. Counter patches mark a tombstone dirty, while event-bearing + // tombstones are discovered directly by the orphan scan. + foreach (var sourceStack in group.Sources) + await _stackRepository.MergeDuplicateStackAsync(group.Target.Id, sourceStack); + + long reassigned = await AwaitWithLockRenewalAsync( + _eventRepository.ReassignStackAsync( + group.Sources.Select(s => s.Id), group.Target.Id, context.CancellationToken), + context); + + foreach (var sourceStack in group.Sources) + { + if (!sourceStack.IsDeleted) + { + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, group.Target.Id, isDeleted: true); + sourceStack.IsDeleted = true; + sourceStack.RedirectToStackId = group.Target.Id; + } + + await _stackRepository.MarkDuplicateStackReconciledAsync(sourceStack); + } + + if (reassigned > 0) + { + _logger.LogInformation( + "Reassigned {EventCount} late event(s) from {SourceStackCount} duplicate stack(s) to {TargetStackId}", + reassigned, + group.Sources.Count, + group.Target.Id); + } + } + } + private Task RenewLockAsync(JobContext context) { _lastRun = _timeProvider.GetUtcNow().UtcDateTime; return context.RenewLockAsync(); } + private async Task AwaitWithLockRenewalAsync(Task operation, JobContext context) + { + while (!operation.IsCompleted) + { + var completed = await Task.WhenAny(operation, Task.Delay(TimeSpan.FromSeconds(30), _timeProvider)); + if (completed == operation) + break; + + await RenewLockAsync(context); + } + + return await operation; + } + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { if (!_lastRun.HasValue) diff --git a/src/Exceptionless.Core/Jobs/StackStatusJob.cs b/src/Exceptionless.Core/Jobs/StackStatusJob.cs index 0f0c8d3d2f..294916e6ce 100644 --- a/src/Exceptionless.Core/Jobs/StackStatusJob.cs +++ b/src/Exceptionless.Core/Jobs/StackStatusJob.cs @@ -1,5 +1,4 @@ -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; @@ -42,10 +41,7 @@ protected override async Task RunInternalAsync(JobContext context) var results = await _stackRepository.GetExpiredSnoozedStatuses(_timeProvider.GetUtcNow().UtcDateTime, o => o.PageLimit(LIMIT)); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { - foreach (var stack in results.Documents) - stack.MarkOpen(); - - await _stackRepository.SaveAsync(results.Documents); + await _stackRepository.MarkOpenAsync(results.Documents.Select(stack => stack.Id)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index 911835df31..f5233b39ea 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -4,12 +4,13 @@ using System.Runtime.Serialization; using System.Text.Json.Serialization; using Exceptionless.Core.Attributes; +using Exceptionless.Core.Serialization; using Foundatio.Repositories.Models; namespace Exceptionless.Core.Models; [DebuggerDisplay("Id={Id} Type={Type} Status={Status} IsDeleted={IsDeleted} Title={Title} TotalOccurrences={TotalOccurrences}")] -public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes, IValidatableObject +public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes, IVersioned, IValidatableObject { /// /// Unique id that identifies a stack. @@ -120,6 +121,36 @@ public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISu public DateTime UpdatedUtc { get; set; } public bool IsDeleted { get; set; } + /// + /// The canonical stack for events that still reference this duplicate stack. + /// This is internal cleanup state and is not part of the public API contract. + /// + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal string? RedirectToStackId { get; set; } + + /// + /// Tracks how many occurrences from each duplicate stack have already been merged. + /// This is internal cleanup state and is not part of the public API contract. + /// + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal IDictionary MergedDuplicateStackTotals { get; set; } = new Dictionary(); + + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal bool NeedsRedirectReconciliation { get; set; } + + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal string ElasticVersion { get; set; } = null!; + + string IVersioned.Version + { + get => ElasticVersion; + set => ElasticVersion = value; + } + public bool AllowNotifications => Status != StackStatus.Fixed && Status != StackStatus.Ignored && Status != StackStatus.Discarded && Status != StackStatus.Snoozed; public static class KnownTypes diff --git a/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs b/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs index c0d067e00f..6a55257bbc 100644 --- a/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs +++ b/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs @@ -111,7 +111,7 @@ public override async Task ProcessBatchAsync(ICollection contexts) } else { - ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, o => o.Cache()); + ctx.Stack = await _stackRepository.GetCanonicalStackAsync(ctx.Event.StackId); if (ctx.Stack is null || ctx.Stack.ProjectId != ctx.Event.ProjectId) { ctx.SetError("Invalid StackId."); @@ -168,8 +168,8 @@ await _publisher.PublishAsync(new EntityChanged } var stacksToSave = stacks.Where(s => s.Value.ShouldSave).Select(kvp => kvp.Value.Stack).ToList(); - if (stacksToSave.Count > 0) - await _stackRepository.SaveAsync(stacksToSave, o => o.Cache().Notifications(false)); // notification will get sent later in the update stats step + foreach (var stack in stacksToSave) + await _stackRepository.AddEventTagsAsync(stack.Id, stack.Tags); // notification will get sent later in the update stats step // Set stack ids after they have been saved and created contexts.ForEach(ctx => diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 02b15bdd89..5d104bf141 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -47,6 +47,8 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map) .Keyword(e => e.SignatureHash, k => k.IgnoreAbove(1024)) .FieldAlias(Alias.SignatureHash, a => a.Path(f => f.SignatureHash)) .Keyword(e => e.DuplicateSignature) + .Keyword(e => e.RedirectToStackId) + .Boolean(e => e.NeedsRedirectReconciliation) .Keyword(e => e.Type, k => k.IgnoreAbove(1024)) .Date(e => e.FirstOccurrence) .FieldAlias(Alias.FirstOccurrence, a => a.Path(f => f.FirstOccurrence)) diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 53fe18ec95..17a9ed40d4 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -213,30 +213,33 @@ public override Task RemoveAllByProjectIdAsync(string organizationId, stri } public Task RemoveAllByStackIdsAsync(string[] stackIds) + => RemoveAllByStackIdsAsync(stackIds, null); + + public Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options) { ArgumentNullException.ThrowIfNull(stackIds); if (stackIds is []) throw new ArgumentOutOfRangeException(nameof(stackIds)); - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Stack(stackIds)); + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Stack(stackIds), options); } - public Task RemoveAllByProjectIdsAsync(string[] projectIds) + public Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null) { ArgumentNullException.ThrowIfNull(projectIds); if (projectIds is []) throw new ArgumentOutOfRangeException(nameof(projectIds)); - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Project(projectIds)); + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Project(projectIds), options); } - public Task RemoveAllByOrganizationIdsAsync(string[] organizationIds) + public Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null) { ArgumentNullException.ThrowIfNull(organizationIds); if (organizationIds is []) throw new ArgumentOutOfRangeException(nameof(organizationIds)); - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationIds)); + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationIds), options); } private async Task RemoveAllIgnoringMissingEventIndexesAsync( @@ -286,23 +289,80 @@ private static bool IsIndexNotFound(Elastic.Transport.Products.Elasticsearch.Err /// Reassigns all events from the source stacks to the target stack using a parameterized /// Painless script (no string interpolation) to prevent script injection. /// - public Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId) + /// + /// Foundatio's update-by-query task cannot be interrupted safely once submitted. The token is + /// observed before submission and by both strict verification reads; the caller must retain its + /// lease until this method returns so cancellation cannot leave an unobserved background write. + /// + public async Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(sourceStackIds); ArgumentException.ThrowIfNullOrEmpty(targetStackId); // Materialize to avoid multiple enumeration and guard against empty; an empty // .Stack() filter would match ALL events and reassign them to the target stack. - var sourceIds = sourceStackIds.ToList(); + var sourceIds = sourceStackIds.Distinct(StringComparer.Ordinal).ToList(); if (sourceIds.Count == 0) - return Task.FromResult(0L); + return 0; + if (sourceIds.Contains(targetStackId, StringComparer.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(sourceStackIds)); - return PatchAllAsync( - q => q.Stack(sourceIds), - new ScriptPatch("ctx._source.stack_id = params.targetStackId") - { - Params = new Dictionary { ["targetStackId"] = targetStackId } - }); + const int maxAttempts = 5; + long remaining = await CountEventsByStackIdsStrictAsync(sourceIds, cancellationToken); + long affected = 0; + + for (int attempt = 1; remaining > 0 && attempt <= maxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + affected += await PatchAllAsync( + q => q.Stack(sourceIds), + new ScriptPatch("ctx._source.stack_id = params.targetStackId") + { + Params = new Dictionary { ["targetStackId"] = targetStackId } + }, + o => o.ImmediateConsistency().Notifications(false)); + + // ScriptPatch uses update-by-query with Conflicts.Proceed. Verify strictly against + // the same alias and retry any documents skipped by version conflicts. + remaining = await CountEventsByStackIdsStrictAsync(sourceIds, cancellationToken); + } + + if (remaining > 0) + throw new DocumentException($"Unable to reassign {remaining} event(s) after {maxAttempts} attempts."); + + return affected; + } + + private async Task CountEventsByStackIdsStrictAsync(IReadOnlyCollection stackIds, CancellationToken cancellationToken) + { + var count = await _configuration.Client.CountAsync(s => s + .Indices(_configuration.Events.Name) + .AllowNoIndices(true) + .IgnoreUnavailable(false) + .ExpandWildcards(ExpandWildcard.All) + .Query(q => q.Terms(t => t + .Field(e => e.StackId) + .Terms(new TermsQueryField(stackIds.Select(id => (FieldValue)id).ToList())))), cancellationToken); + + // A brand-new installation can have no event alias yet. Fall back to every concrete + // version only for that specific state so an alias transition can never look like zero. + if (!count.IsValidResponse && count.ElasticsearchServerError is not null && IsIndexNotFound(count.ElasticsearchServerError)) + { + count = await _configuration.Client.CountAsync(s => s + .Indices($"{_configuration.Events.Name}-v*-*") + .AllowNoIndices(true) + .IgnoreUnavailable(false) + .ExpandWildcards(ExpandWildcard.All) + .Query(q => q.Terms(t => t + .Field(e => e.StackId) + .Terms(new TermsQueryField(stackIds.Select(id => (FieldValue)id).ToList())))), cancellationToken); + } + + if (!count.IsValidResponse || count.Shards.Failed > 0) + throw new DocumentException($"Unable to verify event reassignment through the event alias: {count.DebugInformation}", count.ApiCallDetails.OriginalException); + + return count.Count; } public Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index 2c0e4ea69c..531230f899 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -13,9 +13,10 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); - Task RemoveAllByProjectIdsAsync(string[] projectIds); - Task RemoveAllByOrganizationIdsAsync(string[] organizationIds); - Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId); + Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options); + Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null); + Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null); + Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId, CancellationToken cancellationToken = default); Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); Task GetDistinctOrganizationIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs index f2ec72337a..7a508fb020 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs @@ -7,6 +7,7 @@ namespace Exceptionless.Core.Repositories; public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject { Task GetStackBySignatureHashAsync(string projectId, string signatureHash); + Task GetCanonicalStackAsync(string stackId); Task> GetIdsByQueryAsync(RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null); Task> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null); Task MarkAsRegressedAsync(string stackId); @@ -14,6 +15,12 @@ public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true); Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff); Task> GetSoftDeleted(); + Task> GetRedirectedStacksNeedingReconciliationAsync(); Task SoftDeleteByProjectIdAsync(string organizationId, string projectId); Task> GetDuplicateSignaturesAsync(int maxResults = 10000); + Task AddEventTagsAsync(string stackId, IEnumerable tags); + Task MarkOpenAsync(IEnumerable stackIds); + Task SetDuplicateStackRedirectAsync(Stack sourceStack, string targetStackId, bool isDeleted = false); + Task MarkDuplicateStackReconciledAsync(Stack sourceStack); + Task MergeDuplicateStackAsync(string targetStackId, Stack sourceStack); } diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index adc0524d72..54ed08f8be 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -41,11 +41,39 @@ public Task> GetStacksForCleanupAsync(string organizationId, public Task> GetSoftDeleted() { return FindAsync( - q => q.Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash), + q => q + .FieldEmpty(f => f.RedirectToStackId) + .Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash), o => o.SoftDeleteMode(SoftDeleteQueryMode.DeletedOnly).SearchAfterPaging().PageLimit(500) ); } + public Task> GetRedirectedStacksNeedingReconciliationAsync() + { + return FindAsync( + q => q + .FieldHasValue(f => f.RedirectToStackId) + .FieldEquals(f => f.NeedsRedirectReconciliation, true), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All).SearchAfterPaging().PageLimit(500)); + } + + public override Task RemoveAllByOrganizationIdAsync(string organizationId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + return RemoveAllAsync( + q => q.Organization(organizationId), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + } + + public override Task RemoveAllByProjectIdAsync(string organizationId, string projectId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + ArgumentException.ThrowIfNullOrEmpty(projectId); + return RemoveAllAsync( + q => q.Organization(organizationId).Project(projectId), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + } + public async Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true) { // If total occurrences are zero (stack data was reset), then set first occurrence date @@ -72,7 +100,10 @@ Instant parseDate(def dt) { ctx._source.updated_utc = params.updatedUtc; } -ctx._source.total_occurrences += params.count;"; +ctx._source.total_occurrences += params.count; +if (ctx._source.redirect_to_stack_id != null) { + ctx._source.needs_redirect_reconciliation = true; +}"; var operation = new ScriptPatch(script.TrimScript()) { @@ -128,6 +159,10 @@ Instant parseDate(def dt) { if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) { ctx._source.updated_utc = params.updatedUtc; +} + +if (ctx._source.redirect_to_stack_id != null) { + ctx._source.needs_redirect_reconciliation = true; }"; var operation = new ScriptPatch(script.TrimScript()) @@ -155,7 +190,15 @@ Instant parseDate(def dt) { { string key = GetStackSignatureCacheKey(projectId, signatureHash); var hit = await FindOneAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signatureHash), o => o.Cache(key)); - return hit?.Document; + return hit?.Document is null ? null : await ResolveCanonicalStackAsync(hit.Document); + } + + public async Task GetCanonicalStackAsync(string stackId) + { + ArgumentException.ThrowIfNullOrEmpty(stackId); + + var stack = await GetByIdAsync(stackId, o => o.Cache().SoftDeleteMode(SoftDeleteQueryMode.All)); + return stack is null ? null : await ResolveCanonicalStackAsync(stack); } public Task> GetIdsByQueryAsync(RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null) @@ -165,15 +208,24 @@ public Task> GetIdsByQueryAsync(RepositoryQueryDescriptor(stack => + { + if (stack.Status == StackStatus.Regressed) + return false; + + stack.Status = StackStatus.Regressed; + return true; + }), + o => o.Retry(10)); + } + catch (DocumentNotFoundException) { _logger.LogWarning("Stack {StackId} not found when marking as regressed", stackId); - return; } - - stack.Status = StackStatus.Regressed; - await SaveAsync(stack, o => o.Cache()); } public Task SoftDeleteByProjectIdAsync(string organizationId, string projectId) @@ -189,6 +241,8 @@ public Task SoftDeleteByProjectIdAsync(string organizationId, string proje public async Task> GetDuplicateSignaturesAsync(int maxResults = 10000) { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResults); + // ImmediateConsistency forces a segment refresh before the aggregation so that // any stacks soft-deleted in a previous batch are excluded here. Cost: one refresh // per batch (not per item), equivalent to the original explicit index refresh. @@ -203,6 +257,291 @@ public async Task> GetDuplicateSignaturesAsync(int m return buckets.Select(b => b.Key).ToArray(); } + public async Task AddEventTagsAsync(string stackId, IEnumerable tags) + { + ArgumentException.ThrowIfNullOrEmpty(stackId); + ArgumentNullException.ThrowIfNull(tags); + var tagsToAdd = tags.ToArray(); + + string? redirectToStackId = null; + bool modified = await PatchAsync( + stackId, + new ActionPatch(stack => + { + if (!String.IsNullOrEmpty(stack.RedirectToStackId)) + { + redirectToStackId = stack.RedirectToStackId; + return false; + } + + stack.Tags ??= new TagSet(); + var originalTags = new TagSet(stack.Tags); + stack.Tags.UnionWith(tagsToAdd); + stack.Tags.RemoveExcessTags(); + return !stack.Tags.SetEquals(originalTags); + }), + o => o.Notifications(false).Retry(10)); + + if (String.IsNullOrEmpty(redirectToStackId)) + return modified; + + var canonicalStack = await GetCanonicalStackAsync(redirectToStackId); + return canonicalStack is not null && await AddEventTagsAsync(canonicalStack.Id, tagsToAdd); + } + + public Task MarkOpenAsync(IEnumerable stackIds) + { + ArgumentNullException.ThrowIfNull(stackIds); + var ids = new Ids(stackIds.Distinct(StringComparer.Ordinal)); + if (ids.Count == 0) + return Task.FromResult(0L); + + return PatchAsync( + ids, + new ActionPatch(stack => + { + if (stack is { Status: StackStatus.Open, DateFixed: null, FixedInVersion: null, SnoozeUntilUtc: null }) + return false; + + stack.MarkOpen(); + return true; + }), + o => o.Retry(10)); + } + + public async Task SetDuplicateStackRedirectAsync(Stack sourceStack, string targetStackId, bool isDeleted = false) + { + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + if (String.Equals(sourceStack.Id, targetStackId, StringComparison.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(targetStackId)); + + var canonicalTarget = await GetCanonicalStackAsync(targetStackId) + ?? throw new DocumentNotFoundException(targetStackId); + if (String.Equals(sourceStack.Id, canonicalTarget.Id, StringComparison.Ordinal)) + throw new ArgumentException("A stack redirect cannot create a cycle.", nameof(targetStackId)); + + targetStackId = canonicalTarget.Id; + + if (isDeleted) + { + const string finalizeScript = @" +ctx._source.redirect_to_stack_id = params.targetStackId; +ctx._source.is_deleted = true; +ctx._source.needs_redirect_reconciliation = true; +ctx._source.title = ''; +ctx._source.remove('description'); +ctx._source.signature_info = new HashMap(); +ctx._source.tags = new ArrayList(); +ctx._source.references = new ArrayList(); +ctx._source.remove('fixed_in_version');"; + + await PatchAsync( + sourceStack.Id, + new ScriptPatch(finalizeScript.TrimScript()) + { + Params = new Dictionary { ["targetStackId"] = targetStackId } + }, + o => o.Notifications(false).Retry(10)); + } + else + { + await PatchAsync( + sourceStack.Id, + new PartialPatch(new { redirect_to_stack_id = targetStackId, is_deleted = false, needs_redirect_reconciliation = true }), + o => o.ImmediateConsistency().Notifications(false).Retry(10)); + } + + await Cache.RemoveAsync(GetStackSignatureCacheKey(sourceStack)); + } + + public Task MarkDuplicateStackReconciledAsync(Stack sourceStack) + { + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.RedirectToStackId); + + const string script = @" +Instant parseDate(def dt) { + if (dt != null) { + try { + return Instant.parse(dt); + } catch(DateTimeParseException e) {} + } + return Instant.MIN; +} + +if (ctx._source.needs_redirect_reconciliation == true + && ctx._source.total_occurrences == params.expectedTotalOccurrences + && parseDate(ctx._source.updated_utc).equals(parseDate(params.expectedUpdatedUtc)) + && ctx._source.redirect_to_stack_id == params.expectedTargetStackId) { + ctx._source.needs_redirect_reconciliation = false; +} else { + ctx.op = 'noop'; +}"; + + return PatchAsync( + sourceStack.Id, + new ScriptPatch(script.TrimScript()) + { + Params = new Dictionary + { + ["expectedTotalOccurrences"] = sourceStack.TotalOccurrences, + ["expectedUpdatedUtc"] = sourceStack.UpdatedUtc, + ["expectedTargetStackId"] = sourceStack.RedirectToStackId + } + }, + o => o.Notifications(false).Retry(10)); + } + + public async Task MergeDuplicateStackAsync(string targetStackId, Stack sourceStack) + { + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + if (String.Equals(targetStackId, sourceStack.Id, StringComparison.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(sourceStack)); + + long nestedOccurrenceTotal = sourceStack.MergedDuplicateStackTotals.Values.Sum(total => (long)total); + int sourceOccurrenceTotal = (int)Math.Clamp(sourceStack.TotalOccurrences - nestedOccurrenceTotal, 0, Int32.MaxValue); + var sourceContributions = new Dictionary(sourceStack.MergedDuplicateStackTotals, StringComparer.Ordinal) + { + [sourceStack.Id] = sourceOccurrenceTotal + }; + + // Track every transitive source contribution independently. This makes retries idempotent, + // allows late occurrence deltas, and prevents redirect chains from double-counting nested + // stacks that were already included in the source total. + const string script = @" +Instant parseDate(def dt) { + if (dt != null) { + try { + return Instant.parse(dt); + } catch(DateTimeParseException e) {} + } + return Instant.MIN; +} + +if (ctx._source.merged_duplicate_stack_totals == null) { + ctx._source.merged_duplicate_stack_totals = new HashMap(); +} + +def occurrenceDelta = 0; +for (def contribution : params.sourceContributions.entrySet()) { + def previousContribution = ctx._source.merged_duplicate_stack_totals.containsKey(contribution.getKey()) + ? ctx._source.merged_duplicate_stack_totals[contribution.getKey()] + : 0; + if (contribution.getValue() > previousContribution) { + occurrenceDelta += contribution.getValue() - previousContribution; + } +} + +if (occurrenceDelta <= 0 + && !parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc)) + && !parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence)) + && !parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc)) + && !parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed)) + && !(ctx._source.status == 'open' && params.status != 'open') + && (params.tags == null || ctx._source.tags != null && ctx._source.tags.containsAll(params.tags)) + && (params.references == null || ctx._source.references != null && ctx._source.references.containsAll(params.references)) + && (ctx._source.occurrences_are_critical == true || params.occurrencesAreCritical == false)) { + ctx.op = 'noop'; +} else { + def safeOccurrenceDelta = occurrenceDelta > 0 ? occurrenceDelta : 0; + for (def contribution : params.sourceContributions.entrySet()) { + def previousContribution = ctx._source.merged_duplicate_stack_totals.containsKey(contribution.getKey()) + ? ctx._source.merged_duplicate_stack_totals[contribution.getKey()] + : 0; + if (contribution.getValue() > previousContribution) { + ctx._source.merged_duplicate_stack_totals[contribution.getKey()] = contribution.getValue(); + } + } + def currentTotalOccurrences = ctx._source.total_occurrences == null ? 0 : ctx._source.total_occurrences; + ctx._source.total_occurrences = currentTotalOccurrences + safeOccurrenceDelta; + + if (parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc))) { + ctx._source.created_utc = params.createdUtc; + } + if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence))) { + ctx._source.last_occurrence = params.lastOccurrence; + } + if (parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) { + ctx._source.snooze_until_utc = params.snoozeUntilUtc; + } + if (parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) { + ctx._source.date_fixed = params.dateFixed; + } + if (ctx._source.status == 'open' && params.status != 'open') { + ctx._source.status = params.status; + } + + if (ctx._source.tags == null) { + ctx._source.tags = new ArrayList(); + } + for (int i = 0; i < params.tags.size(); i++) { + if (!ctx._source.tags.contains(params.tags[i])) { + ctx._source.tags.add(params.tags[i]); + } + } + + if (ctx._source.references == null) { + ctx._source.references = new ArrayList(); + } + for (int i = 0; i < params.references.size(); i++) { + if (!ctx._source.references.contains(params.references[i])) { + ctx._source.references.add(params.references[i]); + } + } + + ctx._source.occurrences_are_critical = ctx._source.occurrences_are_critical == true || params.occurrencesAreCritical; +}"; + + var operation = new ScriptPatch(script.TrimScript()) + { + Params = new Dictionary + { + ["sourceContributions"] = sourceContributions, + ["createdUtc"] = sourceStack.CreatedUtc, + ["lastOccurrence"] = sourceStack.LastOccurrence, + ["snoozeUntilUtc"] = sourceStack.SnoozeUntilUtc ?? DateTime.MinValue, + ["dateFixed"] = sourceStack.DateFixed ?? DateTime.MinValue, + ["status"] = sourceStack.Status.ToString().ToLowerInvariant(), + ["tags"] = sourceStack.Tags.ToArray(), + ["references"] = sourceStack.References.ToArray(), + ["occurrencesAreCritical"] = sourceStack.OccurrencesAreCritical + } + }; + + bool modified = await PatchAsync(targetStackId, operation, o => o.Notifications(false).Retry(10)); + if (!modified) + return false; + + var targetStack = await GetByIdAsync(targetStackId, o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + if (targetStack is not null) + await Cache.RemoveAsync(GetStackSignatureCacheKey(targetStack)); + + return true; + } + + private async Task ResolveCanonicalStackAsync(Stack stack) + { + var visitedStackIds = new HashSet(StringComparer.Ordinal) { stack.Id }; + + while (!String.IsNullOrEmpty(stack.RedirectToStackId)) + { + if (!visitedStackIds.Add(stack.RedirectToStackId)) + throw new DocumentException($"Circular stack redirect detected for stack {stack.Id}."); + + stack = await GetByIdAsync( + stack.RedirectToStackId, + o => o.Cache().SoftDeleteMode(SoftDeleteQueryMode.All)) + ?? throw new DocumentNotFoundException(stack.RedirectToStackId); + } + + return stack.IsDeleted ? null : stack; + } + protected override async Task AddDocumentsToCacheAsync(ICollection> findHits, ICommandOptions options, bool isDirtyRead) { await base.AddDocumentsToCacheAsync(findHits, options, isDirtyRead); diff --git a/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs b/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs new file mode 100644 index 0000000000..5dd8f030a2 --- /dev/null +++ b/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs @@ -0,0 +1,7 @@ +namespace Exceptionless.Core.Serialization; + +/// +/// Excludes an internal persistence property from HTTP API serialization. +/// +[AttributeUsage(AttributeTargets.Property)] +internal sealed class JsonIgnoreForExternalSerializationAttribute : Attribute; diff --git a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs index cd27fb785e..872a1afb47 100644 --- a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs +++ b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs @@ -35,6 +35,10 @@ public static JsonSerializerOptions ConfigureExceptionlessDefaults(this JsonSeri public static JsonSerializerOptions ConfigureExceptionlessApiDefaults(this JsonSerializerOptions options) { ConfigureExceptionlessDefaults(options, skipEmptyCollections: false); + + if (options.TypeInfoResolver is DefaultJsonTypeInfoResolver resolver) + resolver.Modifiers.Add(RemoveInternalProperties); + return options; } @@ -73,4 +77,13 @@ private static JsonSerializerOptions ConfigureExceptionlessDefaults(JsonSerializ options.TypeInfoResolver = resolver; return options; } + + private static void RemoveInternalProperties(JsonTypeInfo typeInfo) + { + for (int i = typeInfo.Properties.Count - 1; i >= 0; i--) + { + if (typeInfo.Properties[i].AttributeProvider?.IsDefined(typeof(JsonIgnoreForExternalSerializationAttribute), inherit: true) == true) + typeInfo.Properties.RemoveAt(i); + } + } } diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index 143d15f50a..0ab2d63e52 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -17,7 +17,6 @@ - diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs index c75a98ecf5..f0732d33f4 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs @@ -46,7 +46,7 @@ await _eventRepository.AddAsync( } [Fact] - public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrgRetention() + public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrganizationRetention() { // Arrange // Retention enforcement uses the next plan above the organization's retention: diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index 4aafe47c1d..70eeccfde7 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -3,6 +3,7 @@ using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; @@ -24,6 +25,7 @@ public class CleanupOrphanedDataJobTests : IntegrationTestsBase private readonly IEventRepository _eventRepository; private readonly BillingManager _billingManager; private readonly BillingPlans _plans; + private readonly ExceptionlessElasticConfiguration _configuration; public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -38,6 +40,7 @@ public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory f _eventRepository = GetService(); _billingManager = GetService(); _plans = GetService(); + _configuration = GetService(); } [Fact] @@ -595,6 +598,129 @@ public async Task FixDuplicateStacks_WithDuplicateSignatures_MergesIntoMostPopul Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id))); } + [Fact] + public async Task FixDuplicateStacks_AfterPartialTargetMerge_DoesNotDoubleApplyMetadata() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, targetStack.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, sourceStack.Id), o => o.ImmediateConsistency()); + + // Simulate a prior run that merged the target but failed before hiding the source. + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, sourceStack); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.IncludeSoftDeletes()); + var updatedSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes()); + Assert.NotNull(updatedTarget); + Assert.NotNull(updatedSource); + Assert.Equal(110, updatedTarget.TotalOccurrences); + Assert.False(updatedTarget.IsDeleted); + Assert.True(updatedSource.IsDeleted); + Assert.Equal(110, await _eventRepository.CountAsync(q => q.Stack(targetStack.Id), o => o.ImmediateConsistency())); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id))); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithRedirectedSource_ReassignsLateEvents() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, targetStack.Id, isDeleted: true); + + await _eventRepository.AddAsync( + _eventData.GenerateEvents(10, organization.Id, project.Id, sourceStack.Id), + o => o.ImmediateConsistency()); + + await RefreshDataAsync(); + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(targetStack.Id), o => o.ImmediateConsistency())); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id), o => o.ImmediateConsistency())); + + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(110, updatedTarget.TotalOccurrences); + + var redirectTombstone = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes()); + Assert.NotNull(redirectTombstone); + Assert.True(redirectTombstone.IsDeleted); + Assert.Equal(targetStack.Id, redirectTombstone.RedirectToStackId); + Assert.False(redirectTombstone.NeedsRedirectReconciliation); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithLateCounterAndNoSourceEvents_ReconcilesTombstone() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, sourceStack); + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, targetStack.Id, isDeleted: true); + + await _stackRepository.IncrementEventCounterAsync( + sourceStack.OrganizationId, + sourceStack.ProjectId, + sourceStack.Id, + sourceStack.FirstOccurrence, + sourceStack.LastOccurrence.AddMinutes(1), + 5, + sendNotifications: false); + + var incrementedSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(incrementedSource); + Assert.Equal(15, incrementedSource.TotalOccurrences); + + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id), o => o.ImmediateConsistency())); + + await RefreshDataAsync(); + await _job.RunAsync(TestCancellationToken); + + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(115, updatedTarget.TotalOccurrences); + + var reconciledSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(reconciledSource); + Assert.False(reconciledSource.NeedsRedirectReconciliation); + Assert.Empty((await _stackRepository.GetRedirectedStacksNeedingReconciliationAsync()).Documents); + + DateTime targetUpdatedUtc = updatedTarget.UpdatedUtc; + await _job.RunAsync(TestCancellationToken); + updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(targetUpdatedUtc, updatedTarget.UpdatedUtc); + } + [Fact] public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() { @@ -658,6 +784,48 @@ public async Task FixDuplicateStacks_WithNoEvents_KeepsOldestStack() Assert.True(updatedDuplicate.IsDeleted); } + [Fact] + public async Task FixDuplicateStacks_WithClosedEventIndex_KeepsAllStacksActive() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var occurrenceDate = _configuration.TimeProvider.GetUtcNow().AddDays(-1); + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + await _stackRepository.AddAsync([originalStack, duplicateStack], o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + Enumerable.Range(0, 10).Select(_ => _eventData.GenerateEvent( + organization.Id, project.Id, duplicateStack.Id, occurrenceDate: occurrenceDate)), + o => o.ImmediateConsistency()); + + string eventIndex = _configuration.Events.GetVersionedIndex(occurrenceDate.UtcDateTime); + var closeResponse = await _configuration.Client.Indices.CloseAsync(eventIndex, TestContext.Current.CancellationToken); + Assert.True(closeResponse.IsValidResponse, closeResponse.DebugInformation); + + try + { + // Act + await _job.RunAsync(TestCancellationToken); + } + finally + { + var openResponse = await _configuration.Client.Indices.OpenAsync(eventIndex, TestContext.Current.CancellationToken); + Assert.True(openResponse.IsValidResponse, openResponse.DebugInformation); + } + + // Assert + var stacks = await _stackRepository.GetByIdsAsync( + [originalStack.Id, duplicateStack.Id], + o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(2, stacks.Count); + Assert.All(stacks, stack => Assert.False(stack.IsDeleted)); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id), o => o.ImmediateConsistency())); + } + [Fact] public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() { diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 3dffd0aa6a..dd4466729a 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -334,7 +334,7 @@ public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() await _repository.AddAsync(_eventData.GenerateEvents(5, TestConstants.OrganizationId, TestConstants.ProjectId, stack2.Id), o => o.ImmediateConsistency()); // Act - long affected = await _repository.ReassignStackAsync([stack1.Id], stack2.Id); + long affected = await _repository.ReassignStackAsync([stack1.Id], stack2.Id, TestContext.Current.CancellationToken); // Assert Assert.Equal(10, affected); @@ -346,11 +346,13 @@ public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() } [Fact] - public async Task RemoveAllByProjectIds_WithMatchingEvents_RemovesAll() + public async Task RemoveAllByProjectIds_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string otherProjectId = ObjectId.GenerateNewId().ToString(); await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, otherProjectId, stack.Id), o => o.ImmediateConsistency()); // Act long removed = await _repository.RemoveAllByProjectIdsAsync([TestConstants.ProjectId]); @@ -359,15 +361,18 @@ public async Task RemoveAllByProjectIds_WithMatchingEvents_RemovesAll() Assert.Equal(10, removed); await RefreshDataAsync(); - Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + Assert.Equal(0, await _repository.CountAsync(q => q.Project(TestConstants.ProjectId))); + Assert.Equal(3, await _repository.CountAsync(q => q.Project(otherProjectId))); } [Fact] - public async Task RemoveAllByOrganizationIds_WithMatchingEvents_RemovesAll() + public async Task RemoveAllByOrganizationIds_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string otherOrganizationId = ObjectId.GenerateNewId().ToString(); await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, otherOrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); // Act long removed = await _repository.RemoveAllByOrganizationIdsAsync([TestConstants.OrganizationId]); @@ -376,15 +381,18 @@ public async Task RemoveAllByOrganizationIds_WithMatchingEvents_RemovesAll() Assert.Equal(10, removed); await RefreshDataAsync(); - Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + Assert.Equal(0, await _repository.CountAsync(q => q.Organization(TestConstants.OrganizationId))); + Assert.Equal(3, await _repository.CountAsync(q => q.Organization(otherOrganizationId))); } [Fact] - public async Task RemoveAllByStackIds_WithMatchingEvents_RemovesAll() + public async Task RemoveAllByStackIds_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var otherStack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, otherStack.Id), o => o.ImmediateConsistency()); // Act long removed = await _repository.RemoveAllByStackIdsAsync([stack.Id]); @@ -393,7 +401,8 @@ public async Task RemoveAllByStackIds_WithMatchingEvents_RemovesAll() Assert.Equal(10, removed); await RefreshDataAsync(); - Assert.Equal(0, await _repository.CountAsync(o => o.IncludeSoftDeletes())); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack.Id))); + Assert.Equal(3, await _repository.CountAsync(q => q.Stack(otherStack.Id))); } [Fact] @@ -405,7 +414,7 @@ public async Task ReassignStack_WithEmptySourceIds_ReturnsZeroWithoutModificatio await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); // Act - empty source list must be a no-op; an unchecked empty .Stack() filter would patch ALL events - long affected = await _repository.ReassignStackAsync([], stack2.Id); + long affected = await _repository.ReassignStackAsync([], stack2.Id, TestContext.Current.CancellationToken); // Assert Assert.Equal(0, affected); @@ -443,18 +452,18 @@ public async Task GetDistinctOrganizationIds_WithMultipleOrganizations_ReturnsAl { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); - string org2Id = ObjectId.GenerateNewId().ToString(); + string organization2Id = ObjectId.GenerateNewId().ToString(); await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); - await _repository.AddAsync(_eventData.GenerateEvents(2, org2Id, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, organization2Id, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); // Act var page = await _repository.GetDistinctOrganizationIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); - var orgIds = page.Values; + var organizationIds = page.Values; // Assert - Assert.Equal(orgIds.Count, orgIds.Distinct(StringComparer.Ordinal).Count()); - Assert.Contains(TestConstants.OrganizationId, orgIds); - Assert.Contains(org2Id, orgIds); + Assert.Equal(organizationIds.Count, organizationIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(TestConstants.OrganizationId, organizationIds); + Assert.Contains(organization2Id, organizationIds); } } diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 76270de5c3..8737c5a021 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -7,6 +7,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Repositories; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Options; using Foundatio.Repositories.Utility; using Foundatio.Serializer; @@ -335,4 +336,301 @@ public async Task GetDuplicateSignatures_WithSoftDeletedStacks_ExcludesThem() // The soft-deleted stack should be excluded, leaving only 1 stack with this signature Assert.DoesNotContain($"{uniqueProjectId}:softdelete_sig", duplicates); } + + [Fact] + public async Task GetSoftDeleted_WithRedirect_ExcludesRedirectTombstone() + { + var source = _stackData.GenerateSampleStack(); + source.IsDeleted = true; + source.RedirectToStackId = ObjectId.GenerateNewId().ToString(); + source.NeedsRedirectReconciliation = true; + await _repository.AddAsync(source, o => o.ImmediateConsistency()); + + var softDeleted = await _repository.GetSoftDeleted(); + var redirected = await _repository.GetRedirectedStacksNeedingReconciliationAsync(); + + Assert.DoesNotContain(softDeleted.Documents, stack => stack.Id == source.Id); + Assert.Contains(redirected.Documents, stack => stack.Id == source.Id); + } + + [Fact] + public async Task GetCanonicalStack_WithRedirect_ReturnsActiveTarget() + { + var target = _stackData.GenerateSampleStack(); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.IsDeleted = true; + source.RedirectToStackId = target.Id; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + var canonical = await _repository.GetCanonicalStackAsync(source.Id); + await _repository.AddEventTagsAsync(source.Id, ["redirected-tag"]); + var updatedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + + Assert.NotNull(canonical); + Assert.Equal(target.Id, canonical.Id); + Assert.NotNull(updatedTarget); + Assert.Contains("redirected-tag", updatedTarget.Tags); + } + + [Fact] + public async Task SetDuplicateStackRedirect_WithCycle_Throws() + { + var stackA = _stackData.GenerateSampleStack(); + var stackB = stackA.DeepClone(); + stackB.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([stackA, stackB], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(stackA, stackB.Id); + + await Assert.ThrowsAsync(() => + _repository.SetDuplicateStackRedirectAsync(stackB, stackA.Id)); + + var unchangedTarget = await _repository.GetByIdAsync(stackB.Id, o => o.ImmediateConsistency()); + Assert.NotNull(unchangedTarget); + Assert.Null(unchangedTarget.RedirectToStackId); + } + + [Fact] + public async Task Save_WithStaleVersion_CannotOverwriteDuplicateMerge() + { + var target = _stackData.GenerateSampleStack(); + target.TotalOccurrences = 100; + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.TotalOccurrences = 10; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + var staleTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(staleTarget); + await _repository.MergeDuplicateStackAsync(target.Id, source); + + staleTarget.Title = "stale write"; + await Assert.ThrowsAsync(() => + _repository.SaveAsync(staleTarget, o => o.ImmediateConsistency())); + + var mergedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(mergedTarget); + Assert.Equal(110, mergedTarget.TotalOccurrences); + Assert.Equal(10, mergedTarget.MergedDuplicateStackTotals[source.Id]); + } + + [Fact] + public async Task AddEventTags_WithConcurrentCounterUpdates_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.TotalOccurrences = 0; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll(Enumerable.Range(0, 20).Select(async index => + { + await Task.WhenAll( + _repository.AddEventTagsAsync(stack.Id, [$"tag-{index}"]), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(index + 1), + 1, + sendNotifications: false)); + })); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(20, updated.TotalOccurrences); + Assert.All(Enumerable.Range(0, 20), index => Assert.Contains($"tag-{index}", updated.Tags)); + } + + [Fact] + public async Task MarkOpen_WithConcurrentCounterUpdate_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.Status = StackStatus.Snoozed; + stack.SnoozeUntilUtc = DateTime.UtcNow.AddMinutes(-1); + stack.TotalOccurrences = 10; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll( + _repository.MarkOpenAsync([stack.Id]), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false)); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(StackStatus.Open, updated.Status); + Assert.Null(updated.SnoozeUntilUtc); + Assert.Equal(11, updated.TotalOccurrences); + } + + [Fact] + public async Task MarkAsRegressed_WithConcurrentCounterUpdate_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.Status = StackStatus.Fixed; + stack.TotalOccurrences = 10; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll( + _repository.MarkAsRegressedAsync(stack.Id), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false)); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(StackStatus.Regressed, updated.Status); + Assert.Equal(11, updated.TotalOccurrences); + } + + [Fact] + public async Task MergeDuplicateStack_WithRedirectChain_AppliesOnlyLateDelta() + { + var stackA = _stackData.GenerateSampleStack(); + stackA.TotalOccurrences = 100; + var stackB = stackA.DeepClone(); + stackB.Id = ObjectId.GenerateNewId().ToString(); + stackB.TotalOccurrences = 10; + var stackC = stackA.DeepClone(); + stackC.Id = ObjectId.GenerateNewId().ToString(); + stackC.TotalOccurrences = 50; + await _repository.AddAsync([stackA, stackB, stackC], o => o.ImmediateConsistency()); + + await _repository.MergeDuplicateStackAsync(stackA.Id, stackB); + await _repository.SetDuplicateStackRedirectAsync(stackB, stackA.Id, isDeleted: true); + + stackA = await _repository.GetByIdAsync(stackA.Id, o => o.ImmediateConsistency()) ?? throw new InvalidOperationException(); + await _repository.MergeDuplicateStackAsync(stackC.Id, stackA); + await _repository.SetDuplicateStackRedirectAsync(stackA, stackC.Id, isDeleted: true); + + await _repository.IncrementEventCounterAsync( + stackB.OrganizationId, + stackB.ProjectId, + stackB.Id, + stackB.FirstOccurrence, + stackB.LastOccurrence.AddMinutes(1), + 5, + sendNotifications: false); + stackB = await _repository.GetByIdAsync(stackB.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()) ?? throw new InvalidOperationException(); + await _repository.MergeDuplicateStackAsync(stackC.Id, stackB); + + var canonical = await _repository.GetCanonicalStackAsync(stackB.Id); + var mergedTarget = await _repository.GetByIdAsync(stackC.Id, o => o.ImmediateConsistency()); + Assert.NotNull(canonical); + Assert.Equal(stackC.Id, canonical.Id); + Assert.NotNull(mergedTarget); + Assert.Equal(165, mergedTarget.TotalOccurrences); + Assert.Equal(100, mergedTarget.MergedDuplicateStackTotals[stackA.Id]); + Assert.Equal(15, mergedTarget.MergedDuplicateStackTotals[stackB.Id]); + } + + [Fact] + public async Task RemoveAllByProjectId_WithRedirectTombstone_RemovesAllStacks() + { + string organizationId = ObjectId.GenerateNewId().ToString(); + string projectId = ObjectId.GenerateNewId().ToString(); + var target = _stackData.GenerateStack(generateId: true, organizationId: organizationId, projectId: projectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + await _repository.RemoveAllByProjectIdAsync(organizationId, projectId); + + var remaining = await _repository.GetByIdsAsync([target.Id, source.Id], o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Empty(remaining); + } + + [Fact] + public async Task RemoveAllByOrganizationId_WithRedirectTombstone_RemovesAllStacks() + { + string organizationId = ObjectId.GenerateNewId().ToString(); + string projectId = ObjectId.GenerateNewId().ToString(); + var target = _stackData.GenerateStack(generateId: true, organizationId: organizationId, projectId: projectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + await _repository.RemoveAllByOrganizationIdAsync(organizationId); + + var remaining = await _repository.GetByIdsAsync([target.Id, source.Id], o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Empty(remaining); + } + + [Fact] + public async Task MergeDuplicateStack_WithRepeatedSource_AppliesMetadataOnce() + { + // Arrange + var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + target.CreatedUtc = new DateTime(2026, 1, 2, 0, 0, 0, DateTimeKind.Utc); + target.LastOccurrence = new DateTime(2026, 1, 2, 1, 0, 0, DateTimeKind.Utc); + target.TotalOccurrences = 100; + target.Tags.Add("target"); + target.References.Add("target-reference"); + + var source = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + source.CreatedUtc = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + source.LastOccurrence = new DateTime(2026, 1, 3, 1, 0, 0, DateTimeKind.Utc); + source.TotalOccurrences = 10; + source.Status = StackStatus.Fixed; + source.Tags.Add("source"); + source.References.Add("source-reference"); + source.OccurrencesAreCritical = true; + + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + // Act + DateTime updatedBeforeMerge = target.UpdatedUtc; + await _repository.MergeDuplicateStackAsync(target.Id, source); + + var targetAfterMerge = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(targetAfterMerge); + Assert.True(targetAfterMerge.UpdatedUtc > updatedBeforeMerge); + + await _repository.MergeDuplicateStackAsync(target.Id, source); + var targetAfterNoOp = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(targetAfterNoOp); + Assert.Equal(targetAfterMerge.UpdatedUtc, targetAfterNoOp.UpdatedUtc); + + // A normal full save must preserve the internal retry ledger. + targetAfterMerge.Title = "updated after merge"; + await _repository.SaveAsync(targetAfterMerge, o => o.ImmediateConsistency()); + + await _repository.MergeDuplicateStackAsync(target.Id, source); + + // Assert + var merged = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(merged); + Assert.Equal(110, merged.TotalOccurrences); + Assert.Equal(source.CreatedUtc, merged.CreatedUtc); + Assert.Equal(source.LastOccurrence, merged.LastOccurrence); + Assert.Equal(StackStatus.Fixed, merged.Status); + Assert.Contains("target", merged.Tags); + Assert.Contains("source", merged.Tags); + Assert.Contains("target-reference", merged.References); + Assert.Contains("source-reference", merged.References); + Assert.True(merged.OccurrencesAreCritical); + + // Late in-flight occurrences on the redirected source are merged as a delta. + source.TotalOccurrences = 15; + source.LastOccurrence = source.LastOccurrence.AddMinutes(1); + await _repository.SaveAsync(source, o => o.ImmediateConsistency()); + await _repository.MergeDuplicateStackAsync(target.Id, source); + + merged = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(merged); + Assert.Equal(115, merged.TotalOccurrences); + Assert.Equal(source.LastOccurrence, merged.LastOccurrence); + } } diff --git a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs index 05f47341d4..aa6ffd787d 100644 --- a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs +++ b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs @@ -59,4 +59,25 @@ public void DataDictionaryExtensions_GetValue_PreservesJsonSerializerOptionsOver Assert.Equal("compat-node", environment.MachineName); Assert.Equal("Linux", environment.OSName); } + + [Fact] + public void Stack_InternalCleanupState_IsNotSerializedByApi() + { + var stack = new Stack + { + RedirectToStackId = "target-stack", + MergedDuplicateStackTotals = new Dictionary { ["source-stack"] = 42 }, + NeedsRedirectReconciliation = true + }; + + string apiJson = JsonSerializer.Serialize(stack, new JsonSerializerOptions().ConfigureExceptionlessApiDefaults()); + string storageJson = JsonSerializer.Serialize(stack, new JsonSerializerOptions().ConfigureExceptionlessDefaults()); + + Assert.DoesNotContain("redirect_to_stack_id", apiJson); + Assert.DoesNotContain("merged_duplicate_stack_totals", apiJson); + Assert.DoesNotContain("needs_redirect_reconciliation", apiJson); + Assert.Contains("redirect_to_stack_id", storageJson); + Assert.Contains("merged_duplicate_stack_totals", storageJson); + Assert.Contains("needs_redirect_reconciliation", storageJson); + } } From bd560b0072b5f83341e41cd73699ac2f16deab61 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 15 Jul 2026 21:41:04 -0500 Subject: [PATCH 3/9] test: align cleanup async naming --- .../Jobs/CleanupOrphanedDataJob.cs | 4 +- .../Jobs/CleanupDataJobPaginationTests.cs | 8 +-- .../Jobs/CleanupDataJobTests.cs | 2 +- .../Jobs/CleanupOrphanedDataJobTests.cs | 50 +++++++++---------- .../Repositories/EventRepositoryTests.cs | 18 +++---- .../Repositories/StackRepositoryTests.cs | 26 +++++----- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index a8785970ef..c8c7e34729 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -56,7 +56,7 @@ protected override async Task RunInternalAsync(JobContext context) await DeleteOrphanedEventsByProjectAsync(context); await DeleteOrphanedEventsByOrganizationAsync(context); - await FixDuplicateStacks(context); + await FixDuplicateStacksAsync(context); return JobResult.Success; } @@ -195,7 +195,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) _logger.LogInformation("Completed orphaned events cleanup by organization: deleted {TotalOrphanedEvents} events, checked {TotalOrganizationIds} organizations", totalOrphanedEvents, totalOrganizationIds); } - public async Task FixDuplicateStacks(JobContext context) + public async Task FixDuplicateStacksAsync(JobContext context) { _logger.LogInformation("Getting duplicate stacks"); diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs index f0732d33f4..cea0cbba8e 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs @@ -10,7 +10,7 @@ namespace Exceptionless.Tests.Jobs; public partial class CleanupDataJobTests { [Fact] - public async Task CleanupSoftDeletedOrganizations_WithMultiplePages_RemovesAllData() + public async Task RunAsync_WithMultiplePagesOfSoftDeletedOrganizations_RemovesAllData() { // Arrange var organizations = new List(); @@ -46,7 +46,7 @@ await _eventRepository.AddAsync( } [Fact] - public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrganizationRetention() + public async Task RunAsync_WithMultipleOrganizationRetentionPeriods_RespectsPerOrganizationRetention() { // Arrange // Retention enforcement uses the next plan above the organization's retention: @@ -90,7 +90,7 @@ public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrganiza } [Fact] - public async Task CleanupSoftDeletedStacks_WithMultiplePages_RemovesAllStacks() + public async Task RunAsync_WithMultiplePagesOfSoftDeletedStacks_RemovesAllStacks() { // Arrange var organization = await _organizationRepository.AddAsync( @@ -127,7 +127,7 @@ await _eventRepository.AddAsync( } [Fact] - public async Task EnforceRetention_WithEventsOutsideRetention_DeletesOnlyExpiredEvents() + public async Task RunAsync_WithEventsOutsideRetention_DeletesOnlyExpiredEvents() { // Arrange // FreePlan's 3-day retention is enforced at the next plan threshold (30 days). diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 786391f8bd..e4a76b8440 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -300,7 +300,7 @@ public async Task RunAsync_EventsOutsideRetentionPeriod_RemovesExpiredEvents() } [Fact] - public async Task DeleteOrphanedEventsByStack_WithLargeDataset_DeletesAllOrphanedEvents() + public async Task RunAsync_WithLargeOrphanedEventDataset_DeletesAllOrphanedEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index 70eeccfde7..6b95190c05 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -44,7 +44,7 @@ public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory f } [Fact] - public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents() + public async Task DeleteOrphanedEventsByStackAsync_WithValidStack_DoesNotDeleteEvents() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -72,7 +72,7 @@ public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents } [Fact] - public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() + public async Task DeleteOrphanedEventsByStackAsync_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -106,7 +106,7 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele } [Fact] - public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvents() + public async Task DeleteOrphanedEventsByStackAsync_WithLargeVolume_PreservesAllValidEvents() { // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -136,7 +136,7 @@ public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvent } [Fact] - public async Task DeleteOrphanedEventsByStack_WithManyUniqueOrphanedStacks_DeletesAllOrphanedEvents() + public async Task DeleteOrphanedEventsByStackAsync_WithManyUniqueOrphanedStacks_DeletesAllOrphanedEvents() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -163,7 +163,7 @@ public async Task DeleteOrphanedEventsByStack_WithManyUniqueOrphanedStacks_Delet } [Fact] - public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() + public async Task DeleteOrphanedEventsByStackAsync_WithMultipleValidStacks_PreservesAll() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -196,7 +196,7 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() } [Fact] - public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_OtherTenantUnaffected() + public async Task DeleteOrphanedEventsByStackAsync_WithOrphansInOneTenant_LeavesOtherTenantUnaffected() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -225,7 +225,7 @@ public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_Othe } [Fact] - public async Task DeleteOrphanedEventsByStack_WithSoftDeletedStack_DeletesOrphanedEvents() + public async Task DeleteOrphanedEventsByStackAsync_WithSoftDeletedStack_DeletesOrphanedEvents() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -250,7 +250,7 @@ public async Task DeleteOrphanedEventsByStack_WithSoftDeletedStack_DeletesOrphan } [Fact] - public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteEvents() + public async Task DeleteOrphanedEventsByProjectAsync_WithValidProjects_DoesNotDeleteEvents() { // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -277,7 +277,7 @@ public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteE } [Fact] - public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEventsForMissingProject() + public async Task DeleteOrphanedEventsByProjectAsync_WithOrphanedProject_DeletesEventsForMissingProject() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -306,7 +306,7 @@ public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEvent } [Fact] - public async Task DeleteOrphanedEventsByProject_WithMissingProjects_DeletesOrphanedEvents() + public async Task DeleteOrphanedEventsByProjectAsync_WithMissingProjects_DeletesOrphanedEvents() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -330,7 +330,7 @@ public async Task DeleteOrphanedEventsByProject_WithMissingProjects_DeletesOrpha } [Fact] - public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependent() + public async Task DeleteOrphanedEventsByProjectAsync_WithMultipleTenants_HandlesEachTenantIndependently() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -359,7 +359,7 @@ public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependen } [Fact] - public async Task DeleteOrphanedEventsByProject_WithSoftDeletedProject_DeletesOrphanedEvents() + public async Task DeleteOrphanedEventsByProjectAsync_WithSoftDeletedProject_DeletesOrphanedEvents() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -386,7 +386,7 @@ public async Task DeleteOrphanedEventsByProject_WithSoftDeletedProject_DeletesOr } [Fact] - public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_DoesNotDeleteEvents() + public async Task DeleteOrphanedEventsByOrganizationAsync_WithValidOrganizations_DoesNotDeleteEvents() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -414,7 +414,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_Does } [Fact] - public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_DeletesEventsForMissingOrganization() + public async Task DeleteOrphanedEventsByOrganizationAsync_WithOrphanedOrganization_DeletesEventsForMissingOrganization() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -443,7 +443,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_De } [Fact] - public async Task DeleteOrphanedEventsByOrganization_WithMissingOrganizations_DeletesOrphanedEvents() + public async Task DeleteOrphanedEventsByOrganizationAsync_WithMissingOrganizations_DeletesOrphanedEvents() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -467,7 +467,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithMissingOrganizations_De } [Fact] - public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDeletesOrphanedTenantEvents() + public async Task DeleteOrphanedEventsByOrganizationAsync_WithOneDeletedTenant_OnlyDeletesOrphanedTenantEvents() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -496,7 +496,7 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe } [Fact] - public async Task DeleteOrphanedEventsByOrganization_WithSoftDeletedOrganization_DeletesOrphanedEvents() + public async Task DeleteOrphanedEventsByOrganizationAsync_WithSoftDeletedOrganization_DeletesOrphanedEvents() { // Arrange var validOrganization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -522,7 +522,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithSoftDeletedOrganization } [Fact] - public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly() + public async Task FixDuplicateStacksAsync_WithDuplicatesAcrossTenants_MergesCorrectly() { // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -561,7 +561,7 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly } [Fact] - public async Task FixDuplicateStacks_WithDuplicateSignatures_MergesIntoMostPopularStack() + public async Task FixDuplicateStacksAsync_WithDuplicateSignatures_MergesIntoMostPopularStack() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -599,7 +599,7 @@ public async Task FixDuplicateStacks_WithDuplicateSignatures_MergesIntoMostPopul } [Fact] - public async Task FixDuplicateStacks_AfterPartialTargetMerge_DoesNotDoubleApplyMetadata() + public async Task FixDuplicateStacksAsync_AfterPartialTargetMerge_DoesNotDoubleApplyMetadata() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -635,7 +635,7 @@ public async Task FixDuplicateStacks_AfterPartialTargetMerge_DoesNotDoubleApplyM } [Fact] - public async Task DeleteOrphanedEventsByStack_WithRedirectedSource_ReassignsLateEvents() + public async Task DeleteOrphanedEventsByStackAsync_WithRedirectedSource_ReassignsLateEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -672,7 +672,7 @@ await _eventRepository.AddAsync( } [Fact] - public async Task DeleteOrphanedEventsByStack_WithLateCounterAndNoSourceEvents_ReconcilesTombstone() + public async Task DeleteOrphanedEventsByStackAsync_WithLateCounterAndNoSourceEvents_ReconcilesTombstone() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -722,7 +722,7 @@ await _stackRepository.IncrementEventCounterAsync( } [Fact] - public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() + public async Task FixDuplicateStacksAsync_WithNoDuplicates_DoesNotModifyAnything() { // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); @@ -754,7 +754,7 @@ public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() } [Fact] - public async Task FixDuplicateStacks_WithNoEvents_KeepsOldestStack() + public async Task FixDuplicateStacksAsync_WithNoEvents_KeepsOldestStack() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -785,7 +785,7 @@ public async Task FixDuplicateStacks_WithNoEvents_KeepsOldestStack() } [Fact] - public async Task FixDuplicateStacks_WithClosedEventIndex_KeepsAllStacksActive() + public async Task FixDuplicateStacksAsync_WithClosedEventIndex_KeepsAllStacksActive() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index dd4466729a..17ffbe0a72 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -277,7 +277,7 @@ private async Task CreateDataAsync() } [Fact] - public async Task GetDistinctStackIds_WithMultipleStacks_ReturnsAllUniqueIds() + public async Task GetDistinctStackIdsAsync_WithMultipleStacks_ReturnsAllUniqueIds() { // Arrange var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -297,7 +297,7 @@ public async Task GetDistinctStackIds_WithMultipleStacks_ReturnsAllUniqueIds() } [Fact] - public async Task GetDistinctStackIds_WithPagination_ReturnsAllIds() + public async Task GetDistinctStackIdsAsync_WithPagination_ReturnsAllIds() { // Arrange var stacks = new List(); @@ -324,7 +324,7 @@ public async Task GetDistinctStackIds_WithPagination_ReturnsAllIds() } [Fact] - public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() + public async Task ReassignStackAsync_WithSourceEvents_MovesAllEventsToTarget() { // Arrange var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -346,7 +346,7 @@ public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() } [Fact] - public async Task RemoveAllByProjectIds_WithMixedEvents_RemovesOnlyMatchingEvents() + public async Task RemoveAllByProjectIdsAsync_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -366,7 +366,7 @@ public async Task RemoveAllByProjectIds_WithMixedEvents_RemovesOnlyMatchingEvent } [Fact] - public async Task RemoveAllByOrganizationIds_WithMixedEvents_RemovesOnlyMatchingEvents() + public async Task RemoveAllByOrganizationIdsAsync_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -386,7 +386,7 @@ public async Task RemoveAllByOrganizationIds_WithMixedEvents_RemovesOnlyMatching } [Fact] - public async Task RemoveAllByStackIds_WithMixedEvents_RemovesOnlyMatchingEvents() + public async Task RemoveAllByStackIdsAsync_WithMixedEvents_RemovesOnlyMatchingEvents() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -406,7 +406,7 @@ public async Task RemoveAllByStackIds_WithMixedEvents_RemovesOnlyMatchingEvents( } [Fact] - public async Task ReassignStack_WithEmptySourceIds_ReturnsZeroWithoutModification() + public async Task ReassignStackAsync_WithEmptySourceIds_ReturnsZeroWithoutModification() { // Arrange var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -425,7 +425,7 @@ public async Task ReassignStack_WithEmptySourceIds_ReturnsZeroWithoutModificatio } [Fact] - public async Task GetDistinctProjectIds_WithMultipleProjects_ReturnsAllUniqueIds() + public async Task GetDistinctProjectIdsAsync_WithMultipleProjects_ReturnsAllUniqueIds() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); @@ -448,7 +448,7 @@ public async Task GetDistinctProjectIds_WithMultipleProjects_ReturnsAllUniqueIds } [Fact] - public async Task GetDistinctOrganizationIds_WithMultipleOrganizations_ReturnsAllUniqueIds() + public async Task GetDistinctOrganizationIdsAsync_WithMultipleOrganizations_ReturnsAllUniqueIds() { // Arrange var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 8737c5a021..67a43cec16 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -284,7 +284,7 @@ await _repository.AddAsync( } [Fact] - public async Task GetDuplicateSignatures_WithDuplicates_ReturnsSignatures() + public async Task GetDuplicateSignaturesAsync_WithDuplicates_ReturnsSignatures() { string uniqueProjectId = ObjectId.GenerateNewId().ToString(); var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); @@ -300,7 +300,7 @@ public async Task GetDuplicateSignatures_WithDuplicates_ReturnsSignatures() } [Fact] - public async Task GetDuplicateSignatures_WithNoDuplicates_ReturnsEmpty() + public async Task GetDuplicateSignaturesAsync_WithNoDuplicates_ReturnsEmpty() { // Use a unique project ID to avoid interference from pre-existing sample data string uniqueProjectId = ObjectId.GenerateNewId().ToString(); @@ -319,7 +319,7 @@ public async Task GetDuplicateSignatures_WithNoDuplicates_ReturnsEmpty() } [Fact] - public async Task GetDuplicateSignatures_WithSoftDeletedStacks_ExcludesThem() + public async Task GetDuplicateSignaturesAsync_WithSoftDeletedStacks_ExcludesThem() { // Use a unique project ID to avoid interference from pre-existing sample data string uniqueProjectId = ObjectId.GenerateNewId().ToString(); @@ -354,7 +354,7 @@ public async Task GetSoftDeleted_WithRedirect_ExcludesRedirectTombstone() } [Fact] - public async Task GetCanonicalStack_WithRedirect_ReturnsActiveTarget() + public async Task GetCanonicalStackAsync_WithRedirect_ReturnsActiveTarget() { var target = _stackData.GenerateSampleStack(); var source = target.DeepClone(); @@ -374,7 +374,7 @@ public async Task GetCanonicalStack_WithRedirect_ReturnsActiveTarget() } [Fact] - public async Task SetDuplicateStackRedirect_WithCycle_Throws() + public async Task SetDuplicateStackRedirectAsync_WithCycle_Throws() { var stackA = _stackData.GenerateSampleStack(); var stackB = stackA.DeepClone(); @@ -391,7 +391,7 @@ await Assert.ThrowsAsync(() => } [Fact] - public async Task Save_WithStaleVersion_CannotOverwriteDuplicateMerge() + public async Task SaveAsync_WithStaleVersion_CannotOverwriteDuplicateMerge() { var target = _stackData.GenerateSampleStack(); target.TotalOccurrences = 100; @@ -415,7 +415,7 @@ await Assert.ThrowsAsync(() => } [Fact] - public async Task AddEventTags_WithConcurrentCounterUpdates_PreservesBothChanges() + public async Task AddEventTagsAsync_WithConcurrentCounterUpdates_PreservesBothChanges() { var stack = _stackData.GenerateSampleStack(); stack.TotalOccurrences = 0; @@ -442,7 +442,7 @@ await Task.WhenAll( } [Fact] - public async Task MarkOpen_WithConcurrentCounterUpdate_PreservesBothChanges() + public async Task MarkOpenAsync_WithConcurrentCounterUpdate_PreservesBothChanges() { var stack = _stackData.GenerateSampleStack(); stack.Status = StackStatus.Snoozed; @@ -469,7 +469,7 @@ await Task.WhenAll( } [Fact] - public async Task MarkAsRegressed_WithConcurrentCounterUpdate_PreservesBothChanges() + public async Task MarkAsRegressedAsync_WithConcurrentCounterUpdate_PreservesBothChanges() { var stack = _stackData.GenerateSampleStack(); stack.Status = StackStatus.Fixed; @@ -494,7 +494,7 @@ await Task.WhenAll( } [Fact] - public async Task MergeDuplicateStack_WithRedirectChain_AppliesOnlyLateDelta() + public async Task MergeDuplicateStackAsync_WithRedirectChain_AppliesOnlyLateDelta() { var stackA = _stackData.GenerateSampleStack(); stackA.TotalOccurrences = 100; @@ -535,7 +535,7 @@ await _repository.IncrementEventCounterAsync( } [Fact] - public async Task RemoveAllByProjectId_WithRedirectTombstone_RemovesAllStacks() + public async Task RemoveAllByProjectIdAsync_WithRedirectTombstone_RemovesAllStacks() { string organizationId = ObjectId.GenerateNewId().ToString(); string projectId = ObjectId.GenerateNewId().ToString(); @@ -552,7 +552,7 @@ public async Task RemoveAllByProjectId_WithRedirectTombstone_RemovesAllStacks() } [Fact] - public async Task RemoveAllByOrganizationId_WithRedirectTombstone_RemovesAllStacks() + public async Task RemoveAllByOrganizationIdAsync_WithRedirectTombstone_RemovesAllStacks() { string organizationId = ObjectId.GenerateNewId().ToString(); string projectId = ObjectId.GenerateNewId().ToString(); @@ -569,7 +569,7 @@ public async Task RemoveAllByOrganizationId_WithRedirectTombstone_RemovesAllStac } [Fact] - public async Task MergeDuplicateStack_WithRepeatedSource_AppliesMetadataOnce() + public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnce() { // Arrange var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); From c2bcdc623b4f177b9828e34ba0983ed921d207ce Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 25 Jul 2026 20:36:37 -0500 Subject: [PATCH 4/9] fix: preserve nullable duplicate stack metadata --- src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs | 5 +---- src/Exceptionless.Core/Repositories/StackRepository.cs | 10 ++++++---- .../Repositories/StackRepositoryTests.cs | 6 ++++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index c8c7e34729..8e63f540c5 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -256,9 +256,6 @@ public async Task FixDuplicateStacksAsync(JobContext context) var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); - // We only need to update events if more than one stack has events associated to it. - bool shouldUpdateEvents = eventCountBuckets.Count > 1; - // Default to using the oldest stack. var targetStack = targetCandidates.OrderBy(s => s.CreatedUtc).First(); var duplicateStacks = stacks.Documents.Where(s => s.Id != targetStack.Id).OrderBy(s => s.CreatedUtc).ToList(); @@ -302,7 +299,7 @@ public async Task FixDuplicateStacksAsync(JobContext context) batchProcessed++; long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; - _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); + _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id)); if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(lastStatus) > TimeSpan.FromSeconds(5)) { diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index 54ed08f8be..e1065a6609 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -440,8 +440,8 @@ Instant parseDate(def dt) { if (occurrenceDelta <= 0 && !parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc)) && !parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence)) - && !parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc)) - && !parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed)) + && (params.hasSnoozeUntilUtc == false || !parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) + && (params.hasDateFixed == false || !parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) && !(ctx._source.status == 'open' && params.status != 'open') && (params.tags == null || ctx._source.tags != null && ctx._source.tags.containsAll(params.tags)) && (params.references == null || ctx._source.references != null && ctx._source.references.containsAll(params.references)) @@ -466,10 +466,10 @@ Instant parseDate(def dt) { if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence))) { ctx._source.last_occurrence = params.lastOccurrence; } - if (parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) { + if (params.hasSnoozeUntilUtc && parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) { ctx._source.snooze_until_utc = params.snoozeUntilUtc; } - if (parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) { + if (params.hasDateFixed && parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) { ctx._source.date_fixed = params.dateFixed; } if (ctx._source.status == 'open' && params.status != 'open') { @@ -505,7 +505,9 @@ Instant parseDate(def dt) { ["createdUtc"] = sourceStack.CreatedUtc, ["lastOccurrence"] = sourceStack.LastOccurrence, ["snoozeUntilUtc"] = sourceStack.SnoozeUntilUtc ?? DateTime.MinValue, + ["hasSnoozeUntilUtc"] = sourceStack.SnoozeUntilUtc.HasValue, ["dateFixed"] = sourceStack.DateFixed ?? DateTime.MinValue, + ["hasDateFixed"] = sourceStack.DateFixed.HasValue, ["status"] = sourceStack.Status.ToString().ToLowerInvariant(), ["tags"] = sourceStack.Tags.ToArray(), ["references"] = sourceStack.References.ToArray(), diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 67a43cec16..8851c941f8 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -576,6 +576,8 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc target.CreatedUtc = new DateTime(2026, 1, 2, 0, 0, 0, DateTimeKind.Utc); target.LastOccurrence = new DateTime(2026, 1, 2, 1, 0, 0, DateTimeKind.Utc); target.TotalOccurrences = 100; + target.SnoozeUntilUtc = null; + target.DateFixed = null; target.Tags.Add("target"); target.References.Add("target-reference"); @@ -584,6 +586,8 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc source.LastOccurrence = new DateTime(2026, 1, 3, 1, 0, 0, DateTimeKind.Utc); source.TotalOccurrences = 10; source.Status = StackStatus.Fixed; + source.SnoozeUntilUtc = null; + source.DateFixed = null; source.Tags.Add("source"); source.References.Add("source-reference"); source.OccurrencesAreCritical = true; @@ -616,6 +620,8 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc Assert.Equal(source.CreatedUtc, merged.CreatedUtc); Assert.Equal(source.LastOccurrence, merged.LastOccurrence); Assert.Equal(StackStatus.Fixed, merged.Status); + Assert.Null(merged.SnoozeUntilUtc); + Assert.Null(merged.DateFixed); Assert.Contains("target", merged.Tags); Assert.Contains("source", merged.Tags); Assert.Contains("target-reference", merged.References); From 073b4c2d508b0ca5c8cf96f539e1e5a2abd0cff6 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 9 Aug 2026 21:48:57 -0500 Subject: [PATCH 5/9] fix: harden cleanup reconciliation and coverage --- .../Jobs/CleanupOrphanedDataJob.cs | 51 ++++-- src/Exceptionless.Core/Jobs/StackStatusJob.cs | 8 +- src/Exceptionless.Core/Models/Stack.cs | 5 + .../Repositories/EventRepository.cs | 72 +------- .../Interfaces/IEventRepository.cs | 3 - .../Repositories/StackRepository.cs | 17 +- .../Jobs/CleanupOrphanedDataJobTests.cs | 169 +++++++++++++----- .../Jobs/StackStatusJobTests.cs | 60 +++++++ .../Pipeline/EventPipelineTests.cs | 53 ++++++ .../Repositories/EventRepositoryTests.cs | 62 ++++--- .../Repositories/StackRepositoryTests.cs | 169 +++++++++++++++++- .../Utility/PublicApiCompatibilityTests.cs | 7 +- 12 files changed, 515 insertions(+), 161 deletions(-) create mode 100644 tests/Exceptionless.Tests/Jobs/StackStatusJobTests.cs diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 8e63f540c5..d16a9ed089 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -75,18 +75,21 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) var page = await _eventRepository.GetDistinctStackIdsAsync(500, nextValue, context.CancellationToken); var stackIds = page.Values; - if (stackIds.Count == 0) + if (stackIds.Count is 0) break; nextValue = page.NextValue; hasMore = !String.IsNullOrEmpty(nextValue); totalStackIds += stackIds.Count; + // Keep this destructive existence check on real-time multi-get. An OnlyIds search can + // continue with stale results after a failed refresh, and IsDeleted must be hydrated + // because default soft-delete filtering is applied client-side to multi-get results. var existingStacks = await _stackRepository.GetByIdsAsync(stackIds.ToArray(), o => o.Include(s => s.Id, s => s.IsDeleted)); var existingStackIds = existingStacks.Select(s => s.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingStackIds = stackIds.Where(id => !existingStackIds.Contains(id)).ToArray(); - if (missingStackIds.Length == 0) + if (missingStackIds.Length is 0) continue; // Redirect tombstones are intentionally retained so events from an in-flight ingestion @@ -107,10 +110,12 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) await ReconcileRedirectedStacksAsync(redirectedStackDocuments, context); string[] orphanedStackIds = missingStackIds.Where(id => !redirectedStackIds.Contains(id)).ToArray(); - if (orphanedStackIds.Length == 0) + if (orphanedStackIds.Length is 0) continue; - long deletedCount = await _eventRepository.RemoveAllByStackIdsAsync(orphanedStackIds, o => o.Notifications(false)); + long deletedCount = await _eventRepository.RemoveAllAsync( + q => q.Stack(orphanedStackIds), + o => o.Notifications(false)); totalOrphanedEvents += deletedCount; _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingStackCount} missing stacks out of {StackIdCount} checked", deletedCount, orphanedStackIds.Length, stackIds.Count); @@ -135,7 +140,7 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) var page = await _eventRepository.GetDistinctProjectIdsAsync(500, nextValue, context.CancellationToken); var projectIds = page.Values; - if (projectIds.Count == 0) + if (projectIds.Count is 0) break; nextValue = page.NextValue; @@ -146,10 +151,12 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) var existingProjectIds = existingProjects.Select(p => p.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingProjectIds = projectIds.Where(id => !existingProjectIds.Contains(id)).ToArray(); - if (missingProjectIds.Length == 0) + if (missingProjectIds.Length is 0) continue; - long deletedCount = await _eventRepository.RemoveAllByProjectIdsAsync(missingProjectIds, o => o.Notifications(false)); + long deletedCount = await _eventRepository.RemoveAllAsync( + q => q.Project(missingProjectIds), + o => o.Notifications(false)); totalOrphanedEvents += deletedCount; _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingProjectCount} missing projects out of {ProjectIdCount} checked", deletedCount, missingProjectIds.Length, projectIds.Count); @@ -172,7 +179,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) var page = await _eventRepository.GetDistinctOrganizationIdsAsync(500, nextValue, context.CancellationToken); var organizationIds = page.Values; - if (organizationIds.Count == 0) + if (organizationIds.Count is 0) break; nextValue = page.NextValue; @@ -183,10 +190,12 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) var existingOrganizationIds = existingOrganizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingOrganizationIds = organizationIds.Where(id => !existingOrganizationIds.Contains(id)).ToArray(); - if (missingOrganizationIds.Length == 0) + if (missingOrganizationIds.Length is 0) continue; - long deletedCount = await _eventRepository.RemoveAllByOrganizationIdsAsync(missingOrganizationIds, o => o.Notifications(false)); + long deletedCount = await _eventRepository.RemoveAllAsync( + q => q.Organization(missingOrganizationIds), + o => o.Notifications(false)); totalOrphanedEvents += deletedCount; _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingOrganizationCount} missing organizations out of {OrganizationIdCount} checked", deletedCount, missingOrganizationIds.Length, organizationIds.Count); @@ -212,7 +221,7 @@ public async Task FixDuplicateStacksAsync(JobContext context) while (!context.CancellationToken.IsCancellationRequested) { var duplicateSignatures = await _stackRepository.GetDuplicateSignaturesAsync(); - if (duplicateSignatures.Count == 0) + if (duplicateSignatures.Count is 0) break; batch++; @@ -247,7 +256,7 @@ public async Task FixDuplicateStacksAsync(JobContext context) } var targetCandidates = stacks.Documents.Where(s => String.IsNullOrEmpty(s.RedirectToStackId)).ToList(); - if (targetCandidates.Count == 0) + if (targetCandidates.Count is 0) { _logger.LogError("Did not find a canonical stack for signature {SignatureHash} and project {ProjectId}", signature, projectId); continue; @@ -326,7 +335,7 @@ public async Task FixDuplicateStacksAsync(JobContext context) // If nothing was processed this batch (all errors), stop to avoid an infinite loop // where the same failing signatures are retried indefinitely. - if (batchProcessed == 0) + if (batchProcessed is 0) break; } @@ -397,14 +406,24 @@ private async Task ReconcileRedirectedStacksAsync(IReadOnlyCollection red foreach (var sourceStack in group.Sources) { + var reconciliationSnapshot = sourceStack; if (!sourceStack.IsDeleted) { await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, group.Target.Id, isDeleted: true); - sourceStack.IsDeleted = true; - sourceStack.RedirectToStackId = group.Target.Id; + reconciliationSnapshot = await _stackRepository.GetByIdAsync( + sourceStack.Id, + o => o.IncludeSoftDeletes().ImmediateConsistency()) + ?? throw new DocumentNotFoundException(sourceStack.Id); + + // A counter can land between the first metadata merge and finalization. Merge + // the fresh tombstone before clearing so that late delta is never acknowledged + // without first being applied to the canonical target. + await _stackRepository.MergeDuplicateStackAsync(group.Target.Id, reconciliationSnapshot); } - await _stackRepository.MarkDuplicateStackReconciledAsync(sourceStack); + // Clear against the snapshot that was actually merged. A still-later counter + // update changes updated_utc, fails this compare-and-set, and remains retryable. + await _stackRepository.MarkDuplicateStackReconciledAsync(reconciliationSnapshot); } if (reassigned > 0) diff --git a/src/Exceptionless.Core/Jobs/StackStatusJob.cs b/src/Exceptionless.Core/Jobs/StackStatusJob.cs index 294916e6ce..8c0541759e 100644 --- a/src/Exceptionless.Core/Jobs/StackStatusJob.cs +++ b/src/Exceptionless.Core/Jobs/StackStatusJob.cs @@ -38,9 +38,12 @@ protected override async Task RunInternalAsync(JobContext context) _logger.LogTrace("Start save stack event counts"); // Get list of stacks where snooze has expired - var results = await _stackRepository.GetExpiredSnoozedStatuses(_timeProvider.GetUtcNow().UtcDateTime, o => o.PageLimit(LIMIT)); + var results = await _stackRepository.GetExpiredSnoozedStatuses( + _timeProvider.GetUtcNow().UtcDateTime, + o => o.SearchAfterPaging().PageLimit(LIMIT)); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await context.RenewLockAsync(); await _stackRepository.MarkOpenAsync(results.Documents.Select(stack => stack.Id)); // Sleep so we are not hammering the backend. @@ -48,9 +51,6 @@ protected override async Task RunInternalAsync(JobContext context) if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync()) break; - - if (results.Documents.Count > 0) - await context.RenewLockAsync(); } _logger.LogTrace("Finished save stack event counts"); diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index f5233b39ea..cdc358616f 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -137,6 +137,11 @@ public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISu [JsonIgnoreForExternalSerialization] internal IDictionary MergedDuplicateStackTotals { get; set; } = new Dictionary(); + /// + /// Marks a redirect tombstone whose counters changed after its events may already have been + /// reassigned. With no source events left, the orphan scan cannot rediscover that late update; + /// this durable marker lets cleanup replay the idempotent contribution ledger instead. + /// [JsonInclude] [JsonIgnoreForExternalSerialization] internal bool NeedsRedirectReconciliation { get; set; } diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 17a9ed40d4..2740fdbdf7 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -2,7 +2,6 @@ using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.Aggregations; using Elastic.Clients.Elasticsearch.QueryDsl; -using Elastic.Transport; using Elastic.Transport.Products.Elasticsearch; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; @@ -83,7 +82,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, if (!String.IsNullOrEmpty(clientIpAddress)) query = query.FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); - return RemoveAllIgnoringMissingEventIndexesAsync(q => query, options); + return RemoveAllAsync(q => query, options); } public Task> GetByReferenceIdAsync(string projectId, string referenceId) @@ -197,76 +196,13 @@ public override Task> GetByProjectIdAsync(string pr return FindAsync(q => q.Project(projectId).SortDescending(e => e.Date).SortDescending(e => e.Id), options); } - public override Task RemoveAllByOrganizationIdAsync(string organizationId) - { - ArgumentException.ThrowIfNullOrEmpty(organizationId); - - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId)); - } - - public override Task RemoveAllByProjectIdAsync(string organizationId, string projectId) - { - ArgumentException.ThrowIfNullOrEmpty(organizationId); - ArgumentException.ThrowIfNullOrEmpty(projectId); - - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId).Project(projectId)); - } - public Task RemoveAllByStackIdsAsync(string[] stackIds) - => RemoveAllByStackIdsAsync(stackIds, null); - - public Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options) { ArgumentNullException.ThrowIfNull(stackIds); if (stackIds is []) throw new ArgumentOutOfRangeException(nameof(stackIds)); - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Stack(stackIds), options); - } - - public Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null) - { - ArgumentNullException.ThrowIfNull(projectIds); - if (projectIds is []) - throw new ArgumentOutOfRangeException(nameof(projectIds)); - - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Project(projectIds), options); - } - - public Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null) - { - ArgumentNullException.ThrowIfNull(organizationIds); - if (organizationIds is []) - throw new ArgumentOutOfRangeException(nameof(organizationIds)); - - return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationIds), options); - } - - private async Task RemoveAllIgnoringMissingEventIndexesAsync( - RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null) - { - try - { - return await RemoveAllAsync(query, options); - } - catch (RepositoryException ex) when (IsIndexNotFound(ex.InnerException as TransportException)) - { - return 0; - } - catch (TransportException ex) when (IsIndexNotFound(ex)) - { - return 0; - } - } - - private static bool IsIndexNotFound(TransportException? ex) - { - if (ex?.ApiCallDetails?.HttpStatusCode != 404) - return false; - - return ex.ApiCallDetails.ProductError is ElasticsearchServerError serverError - ? IsIndexNotFound(serverError) - : ex.DebugInformation.Contains("index_not_found_exception", StringComparison.Ordinal); + return RemoveAllAsync(q => q.Stack(stackIds)); } private static bool IsIndexNotFound(ElasticsearchServerError serverError) @@ -302,11 +238,13 @@ public async Task ReassignStackAsync(IEnumerable sourceStackIds, s // Materialize to avoid multiple enumeration and guard against empty; an empty // .Stack() filter would match ALL events and reassign them to the target stack. var sourceIds = sourceStackIds.Distinct(StringComparer.Ordinal).ToList(); - if (sourceIds.Count == 0) + if (sourceIds.Count is 0) return 0; if (sourceIds.Contains(targetStackId, StringComparer.Ordinal)) throw new ArgumentException("Source and target stack ids must be different.", nameof(sourceStackIds)); + cancellationToken.ThrowIfCancellationRequested(); + const int maxAttempts = 5; long remaining = await CountEventsByStackIdsStrictAsync(sourceIds, cancellationToken); long affected = 0; diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index 531230f899..e570ddefb5 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -13,9 +13,6 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); - Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options); - Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null); - Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null); Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId, CancellationToken cancellationToken = default); Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index e1065a6609..c5895ef2c9 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -76,6 +76,8 @@ public override Task RemoveAllByProjectIdAsync(string organizationId, stri public async Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true) { + // An in-flight counter update can reach a redirect tombstone after event reassignment. + // Mark it dirty because no remaining source event may exist for the orphan scan to find. // If total occurrences are zero (stack data was reset), then set first occurrence date // Only update the LastOccurrence if the new date is greater than the existing date. const string script = @" @@ -135,6 +137,8 @@ Instant parseDate(def dt) { public async Task SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true) { + // Keep redirected stacks discoverable when a late stats repair changes their counters + // after all source events have already moved to the canonical stack. const string script = @" Instant parseDate(def dt) { if (dt != null) { @@ -293,7 +297,7 @@ public Task MarkOpenAsync(IEnumerable stackIds) { ArgumentNullException.ThrowIfNull(stackIds); var ids = new Ids(stackIds.Distinct(StringComparer.Ordinal)); - if (ids.Count == 0) + if (ids.Count is 0) return Task.FromResult(0L); return PatchAsync( @@ -362,6 +366,8 @@ public Task MarkDuplicateStackReconciledAsync(Stack sourceStack) ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); ArgumentException.ThrowIfNullOrEmpty(sourceStack.RedirectToStackId); + // Clear the dirty marker only if no concurrent counter update or redirect change landed + // after the caller read this tombstone. Otherwise the next cleanup pass retries it. const string script = @" Instant parseDate(def dt) { if (dt != null) { @@ -374,6 +380,8 @@ Instant parseDate(def dt) { if (ctx._source.needs_redirect_reconciliation == true && ctx._source.total_occurrences == params.expectedTotalOccurrences + && parseDate(ctx._source.first_occurrence).equals(parseDate(params.expectedFirstOccurrence)) + && parseDate(ctx._source.last_occurrence).equals(parseDate(params.expectedLastOccurrence)) && parseDate(ctx._source.updated_utc).equals(parseDate(params.expectedUpdatedUtc)) && ctx._source.redirect_to_stack_id == params.expectedTargetStackId) { ctx._source.needs_redirect_reconciliation = false; @@ -388,6 +396,8 @@ Instant parseDate(def dt) { Params = new Dictionary { ["expectedTotalOccurrences"] = sourceStack.TotalOccurrences, + ["expectedFirstOccurrence"] = sourceStack.FirstOccurrence, + ["expectedLastOccurrence"] = sourceStack.LastOccurrence, ["expectedUpdatedUtc"] = sourceStack.UpdatedUtc, ["expectedTargetStackId"] = sourceStack.RedirectToStackId } @@ -439,6 +449,7 @@ Instant parseDate(def dt) { if (occurrenceDelta <= 0 && !parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc)) + && !parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.firstOccurrence)) && !parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence)) && (params.hasSnoozeUntilUtc == false || !parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) && (params.hasDateFixed == false || !parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) @@ -463,6 +474,9 @@ Instant parseDate(def dt) { if (parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc))) { ctx._source.created_utc = params.createdUtc; } + if (parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.firstOccurrence))) { + ctx._source.first_occurrence = params.firstOccurrence; + } if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence))) { ctx._source.last_occurrence = params.lastOccurrence; } @@ -503,6 +517,7 @@ Instant parseDate(def dt) { { ["sourceContributions"] = sourceContributions, ["createdUtc"] = sourceStack.CreatedUtc, + ["firstOccurrence"] = sourceStack.FirstOccurrence, ["lastOccurrence"] = sourceStack.LastOccurrence, ["snoozeUntilUtc"] = sourceStack.SnoozeUntilUtc ?? DateTime.MinValue, ["hasSnoozeUntilUtc"] = sourceStack.SnoozeUntilUtc.HasValue, diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index 6b95190c05..2123d1369b 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -71,6 +71,29 @@ public async Task DeleteOrphanedEventsByStackAsync_WithValidStack_DoesNotDeleteE Assert.Equal(200, totalCount); } + [Fact] + public async Task DeleteOrphanedEventsByStackAsync_WithUnrefreshedValidStack_PreservesEvents() + { + // Arrange + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + var stack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + await _stackRepository.AddAsync(stack); // Deliberately leave the stack invisible to search. + await _eventRepository.AddAsync( + _eventData.GenerateEvents(5, organization.Id, project.Id, stack.Id), + o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.NotNull(await _stackRepository.GetByIdAsync(stack.Id)); + Assert.Equal(5, await _eventRepository.CountAsync(q => q.Stack(stack.Id), o => o.ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByStackAsync_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() { @@ -292,8 +315,7 @@ public async Task DeleteOrphanedEventsByProjectAsync_WithOrphanedProject_Deletes var validEvents = _eventData.GenerateEvents(75, organization1.Id, validProject.Id, validStack.Id).ToList(); string fakeProjectId = ObjectId.GenerateNewId().ToString(); - string fakeStackId = ObjectId.GenerateNewId().ToString(); - var orphanedEvents = _eventData.GenerateEvents(50, organization2.Id, fakeProjectId, fakeStackId).ToList(); + var orphanedEvents = _eventData.GenerateEvents(50, organization2.Id, fakeProjectId, validStack.Id).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); @@ -303,6 +325,8 @@ public async Task DeleteOrphanedEventsByProjectAsync_WithOrphanedProject_Deletes // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(75, totalAfter); + Assert.Equal(75, await _eventRepository.CountAsync(q => q.Project(validProject.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Project(fakeProjectId))); } [Fact] @@ -345,8 +369,7 @@ public async Task DeleteOrphanedEventsByProjectAsync_WithMultipleTenants_Handles var validEvents = _eventData.GenerateEvents(60, organization1.Id, project1.Id, stack1.Id).ToList(); string nonExistentProjectId = ObjectId.GenerateNewId().ToString(); - string fakeStackId = ObjectId.GenerateNewId().ToString(); - var orphanedEvents = _eventData.GenerateEvents(40, organization2.Id, nonExistentProjectId, fakeStackId).ToList(); + var orphanedEvents = _eventData.GenerateEvents(40, organization2.Id, nonExistentProjectId, stack1.Id).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); @@ -356,6 +379,8 @@ public async Task DeleteOrphanedEventsByProjectAsync_WithMultipleTenants_Handles // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(60, totalAfter); + Assert.Equal(60, await _eventRepository.CountAsync(q => q.Project(project1.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Project(nonExistentProjectId))); } [Fact] @@ -428,9 +453,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync_WithOrphanedOrganizati var validEvents = _eventData.GenerateEvents(100, organization1.Id, project.Id, stack.Id).ToList(); string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); - string fakeProjectId = ObjectId.GenerateNewId().ToString(); - string fakeStackId = ObjectId.GenerateNewId().ToString(); - var orphanedEvents = _eventData.GenerateEvents(50, fakeOrganizationId, fakeProjectId, fakeStackId).ToList(); + var orphanedEvents = _eventData.GenerateEvents(50, fakeOrganizationId, project.Id, stack.Id).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); @@ -440,6 +463,8 @@ public async Task DeleteOrphanedEventsByOrganizationAsync_WithOrphanedOrganizati // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); + Assert.Equal(100, await _eventRepository.CountAsync(q => q.Organization(organization1.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Organization(fakeOrganizationId))); } [Fact] @@ -481,9 +506,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync_WithOneDeletedTenant_O var validEvents = _eventData.GenerateEvents(120, organization1.Id, project1.Id, stack1.Id).ToList(); string ghostOrganizationId = ObjectId.GenerateNewId().ToString(); - string ghostProjectId = ObjectId.GenerateNewId().ToString(); - string ghostStackId = ObjectId.GenerateNewId().ToString(); - var ghostEvents = _eventData.GenerateEvents(80, ghostOrganizationId, ghostProjectId, ghostStackId).ToList(); + var ghostEvents = _eventData.GenerateEvents(80, ghostOrganizationId, project1.Id, stack1.Id).ToList(); await _eventRepository.AddAsync(validEvents.Concat(ghostEvents), o => o.ImmediateConsistency()); @@ -493,6 +516,8 @@ public async Task DeleteOrphanedEventsByOrganizationAsync_WithOneDeletedTenant_O // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(120, totalAfter); + Assert.Equal(120, await _eventRepository.CountAsync(q => q.Organization(organization1.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Organization(ghostOrganizationId))); } [Fact] @@ -522,42 +547,53 @@ public async Task DeleteOrphanedEventsByOrganizationAsync_WithSoftDeletedOrganiz } [Fact] - public async Task FixDuplicateStacksAsync_WithDuplicatesAcrossTenants_MergesCorrectly() + public async Task FixDuplicateStacksAsync_WithDuplicateGroupsAcrossOrganizations_MergesWithinEachProject() { // Arrange - var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); - await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); - var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); - await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); const string signatureHash = "abc123def456"; - var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); - stack1.CreatedUtc = DateTime.UtcNow.AddDays(-10); - stack1.TotalOccurrences = 5; - var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); - stack2.CreatedUtc = DateTime.UtcNow.AddDays(-5); - stack2.TotalOccurrences = 10; - await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + var project1Stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id, signatureHash: signatureHash); + var project1Stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id, signatureHash: signatureHash); + var project2Stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id, signatureHash: signatureHash); + var project2Stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id, signatureHash: signatureHash); + await _stackRepository.AddAsync( + [project1Stack1, project1Stack2, project2Stack1, project2Stack2], + o => o.ImmediateConsistency()); - var events1 = _eventData.GenerateEvents(3, organization.Id, project.Id, stack1.Id).ToList(); - var events2 = _eventData.GenerateEvents(7, organization.Id, project.Id, stack2.Id).ToList(); - await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); + var project1Events = _eventData.GenerateEvents(3, organization1.Id, project1.Id, project1Stack1.Id) + .Concat(_eventData.GenerateEvents(7, organization1.Id, project1.Id, project1Stack2.Id)); + var project2Events = _eventData.GenerateEvents(4, organization2.Id, project2.Id, project2Stack1.Id) + .Concat(_eventData.GenerateEvents(6, organization2.Id, project2.Id, project2Stack2.Id)); + await _eventRepository.AddAsync(project1Events.Concat(project2Events), o => o.ImmediateConsistency()); // Act await _job.RunAsync(TestCancellationToken); // Assert await RefreshDataAsync(); - var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); - var activeStacks = allStacks.Documents.Where(s => !s.IsDeleted).ToList(); - var deletedStacks = allStacks.Documents.Where(s => s.IsDeleted).ToList(); - Assert.Single(activeStacks); - Assert.Single(deletedStacks); - - var allEvents = await _eventRepository.GetAllAsync(); - Assert.Equal(10, allEvents.Total); - Assert.All(allEvents.Documents, e => Assert.Equal(activeStacks[0].Id, e.StackId)); + var allStacks = await _stackRepository.GetByIdsAsync( + [project1Stack1.Id, project1Stack2.Id, project2Stack1.Id, project2Stack2.Id], + o => o.IncludeSoftDeletes()); + Assert.Equal(4, allStacks.Count); + + var activeProject1Stack = Assert.Single(allStacks, stack => stack.ProjectId == project1.Id && !stack.IsDeleted); + var activeProject2Stack = Assert.Single(allStacks, stack => stack.ProjectId == project2.Id && !stack.IsDeleted); + Assert.Single(allStacks, stack => stack.ProjectId == project1.Id && stack.IsDeleted); + Assert.Single(allStacks, stack => stack.ProjectId == project2.Id && stack.IsDeleted); + + var project1EventsAfterMerge = await _eventRepository.FindAsync(q => q.Project(project1.Id)); + var project2EventsAfterMerge = await _eventRepository.FindAsync(q => q.Project(project2.Id)); + Assert.Equal(10, project1EventsAfterMerge.Total); + Assert.Equal(10, project2EventsAfterMerge.Total); + Assert.All(project1EventsAfterMerge.Documents, e => Assert.Equal(activeProject1Stack.Id, e.StackId)); + Assert.All(project2EventsAfterMerge.Documents, e => Assert.Equal(activeProject2Stack.Id, e.StackId)); } [Fact] @@ -687,11 +723,12 @@ public async Task DeleteOrphanedEventsByStackAsync_WithLateCounterAndNoSourceEve await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, sourceStack); await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, targetStack.Id, isDeleted: true); + DateTime earlierFirstOccurrence = sourceStack.FirstOccurrence.AddMinutes(-1); await _stackRepository.IncrementEventCounterAsync( sourceStack.OrganizationId, sourceStack.ProjectId, sourceStack.Id, - sourceStack.FirstOccurrence, + earlierFirstOccurrence, sourceStack.LastOccurrence.AddMinutes(1), 5, sendNotifications: false); @@ -708,6 +745,7 @@ await _stackRepository.IncrementEventCounterAsync( var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); Assert.NotNull(updatedTarget); Assert.Equal(115, updatedTarget.TotalOccurrences); + Assert.Equal(earlierFirstOccurrence, updatedTarget.FirstOccurrence); var reconciledSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.NotNull(reconciledSource); @@ -785,7 +823,7 @@ public async Task FixDuplicateStacksAsync_WithNoEvents_KeepsOldestStack() } [Fact] - public async Task FixDuplicateStacksAsync_WithClosedEventIndex_KeepsAllStacksActive() + public async Task FixDuplicateStacksAsync_WithClosedEventIndex_RecoversAfterIndexReopens() { // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); @@ -794,8 +832,12 @@ public async Task FixDuplicateStacksAsync_WithClosedEventIndex_KeepsAllStacksAct var occurrenceDate = _configuration.TimeProvider.GetUtcNow().AddDays(-1); var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + originalStack.CreatedUtc = occurrenceDate.UtcDateTime.AddMinutes(-2); + originalStack.TotalOccurrences = 100; var duplicateStack = originalStack.DeepClone(); duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + duplicateStack.CreatedUtc = occurrenceDate.UtcDateTime.AddMinutes(-1); + duplicateStack.TotalOccurrences = 10; await _stackRepository.AddAsync([originalStack, duplicateStack], o => o.ImmediateConsistency()); await _eventRepository.AddAsync( Enumerable.Range(0, 10).Select(_ => _eventData.GenerateEvent( @@ -810,6 +852,21 @@ await _eventRepository.AddAsync( { // Act await _job.RunAsync(TestCancellationToken); + + // Assert the failed strict event read leaves a durable, active redirect. + var preservedOriginal = await _stackRepository.GetByIdAsync( + originalStack.Id, + o => o.IncludeSoftDeletes().ImmediateConsistency()); + var preservedDuplicate = await _stackRepository.GetByIdAsync( + duplicateStack.Id, + o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(preservedOriginal); + Assert.NotNull(preservedDuplicate); + Assert.False(preservedOriginal.IsDeleted); + Assert.False(preservedDuplicate.IsDeleted); + Assert.Equal(originalStack.Id, preservedDuplicate.RedirectToStackId); + Assert.True(preservedDuplicate.NeedsRedirectReconciliation); + Assert.Equal(100, preservedOriginal.TotalOccurrences); } finally { @@ -817,13 +874,30 @@ await _eventRepository.AddAsync( Assert.True(openResponse.IsValidResponse, openResponse.DebugInformation); } + await RefreshDataAsync(); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id), o => o.ImmediateConsistency())); + + // Act: a later cleanup pass must converge without losing or double-counting data. + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + // Assert - var stacks = await _stackRepository.GetByIdsAsync( - [originalStack.Id, duplicateStack.Id], + var recoveredOriginal = await _stackRepository.GetByIdAsync( + originalStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); - Assert.Equal(2, stacks.Count); - Assert.All(stacks, stack => Assert.False(stack.IsDeleted)); - Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id), o => o.ImmediateConsistency())); + var recoveredDuplicate = await _stackRepository.GetByIdAsync( + duplicateStack.Id, + o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(recoveredOriginal); + Assert.NotNull(recoveredDuplicate); + Assert.False(recoveredOriginal.IsDeleted); + Assert.Equal(110, recoveredOriginal.TotalOccurrences); + Assert.Equal(10, recoveredOriginal.MergedDuplicateStackTotals[duplicateStack.Id]); + Assert.True(recoveredDuplicate.IsDeleted); + Assert.Equal(originalStack.Id, recoveredDuplicate.RedirectToStackId); + Assert.False(recoveredDuplicate.NeedsRedirectReconciliation); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(originalStack.Id), o => o.ImmediateConsistency())); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id), o => o.ImmediateConsistency())); } [Fact] @@ -840,9 +914,12 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); var validEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id, validStack.Id).ToList(); - var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, ObjectId.GenerateNewId().ToString()).ToList(); - var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); - var orphanedByOrganization = _eventData.GenerateEvents(25, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); + string missingStackId = ObjectId.GenerateNewId().ToString(); + string missingProjectId = ObjectId.GenerateNewId().ToString(); + string missingOrganizationId = ObjectId.GenerateNewId().ToString(); + var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, missingStackId).ToList(); + var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, missingProjectId, validStack.Id).ToList(); + var orphanedByOrganization = _eventData.GenerateEvents(25, missingOrganizationId, project1.Id, validStack.Id).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedByStack).Concat(orphanedByProject).Concat(orphanedByOrganization), o => o.ImmediateConsistency()); @@ -855,6 +932,10 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(missingStackId))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Project(missingProjectId))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Organization(missingOrganizationId))); + Assert.Equal(100, await _eventRepository.CountAsync(q => q.Stack(validStack.Id))); } [Fact] diff --git a/tests/Exceptionless.Tests/Jobs/StackStatusJobTests.cs b/tests/Exceptionless.Tests/Jobs/StackStatusJobTests.cs new file mode 100644 index 0000000000..63d14b78b9 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/StackStatusJobTests.cs @@ -0,0 +1,60 @@ +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Tests.Utility; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class StackStatusJobTests : IntegrationTestsBase +{ + private readonly StackStatusJob _job; + private readonly StackData _stackData; + private readonly IStackRepository _stackRepository; + + public StackStatusJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _job = GetService(); + _stackData = GetService(); + _stackRepository = GetService(); + } + + [Fact] + public async Task RunAsync_WithMoreThanOnePageOfExpiredSnoozedStacks_OpensEveryStack() + { + // Arrange + DateTime utcNow = TimeProvider.GetUtcNow().UtcDateTime; + var stacks = _stackData.GenerateStacks( + 201, + generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId) + .ToList(); + foreach (var stack in stacks) + { + stack.Status = StackStatus.Snoozed; + stack.SnoozeUntilUtc = utcNow.AddMinutes(-1); + stack.DateFixed = utcNow.AddDays(-1); + stack.FixedInVersion = "1.0.0"; + } + + await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var updatedStacks = await _stackRepository.GetByIdsAsync( + stacks.Select(stack => stack.Id).ToArray(), + o => o.ImmediateConsistency()); + Assert.Equal(201, updatedStacks.Count); + Assert.All(updatedStacks, stack => + { + Assert.Equal(StackStatus.Open, stack.Status); + Assert.Null(stack.SnoozeUntilUtc); + Assert.Null(stack.DateFixed); + Assert.Null(stack.FixedInVersion); + }); + } +} diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index d7c94b962e..c565c3bb26 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -17,8 +17,10 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Utility; using Foundatio.Serializer; using Foundatio.Storage; +using Foundatio.Utility; using McSherry.SemanticVersioning; using Xunit; using DataDictionary = Exceptionless.Core.Models.DataDictionary; @@ -30,6 +32,7 @@ public sealed class EventPipelineTests : IntegrationTestsBase private readonly EventPipeline _pipeline; private readonly EventData _eventData; private readonly IEventRepository _eventRepository; + private readonly StackData _stackData; private readonly IStackRepository _stackRepository; private readonly OrganizationData _organizationData; private readonly IOrganizationRepository _organizationRepository; @@ -45,6 +48,7 @@ public EventPipelineTests(ITestOutputHelper output, AppWebHostFactory factory) : { _eventData = GetService(); _eventRepository = GetService(); + _stackData = GetService(); _stackRepository = GetService(); _organizationData = GetService(); _organizationRepository = GetService(); @@ -625,6 +629,55 @@ public async Task SyncStackTagsAsync() Assert.Equal(new[] { Tag1, Tag2 }, stack.Tags.ToArray()); } + [Fact] + public async Task RunAsync_WithRedirectedStackId_PersistsEventAndTagsOnCanonicalStack() + { + // Arrange + const string RedirectedTag = "redirected-pipeline-tag"; + var target = _stackData.GenerateStack( + generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _stackRepository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _stackRepository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + var ev = _eventData.GenerateEvent( + stackId: source.Id, + projectId: TestConstants.ProjectId, + organizationId: TestConstants.OrganizationId, + generateTags: false, + generateData: false, + occurrenceDate: DateTime.UtcNow); + ev.Tags ??= []; + ev.Tags.Add(RedirectedTag); + + // Act + var context = await _pipeline.RunAsync( + ev, + _organizationData.GenerateSampleOrganization(_billingManager, _plans), + _projectData.GenerateSampleProject()); + await RefreshDataAsync(); + + // Assert + Assert.False(context.HasError, context.ErrorMessage); + Assert.Equal(target.Id, context.Stack?.Id); + + var storedEvent = await _eventRepository.GetByIdAsync(ev.Id); + var updatedTarget = await _stackRepository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + var redirectTombstone = await _stackRepository.GetByIdAsync( + source.Id, + o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(storedEvent); + Assert.Equal(target.Id, storedEvent.StackId); + Assert.NotNull(updatedTarget); + Assert.Contains(RedirectedTag, updatedTarget.Tags); + Assert.NotNull(redirectTombstone); + Assert.Equal(target.Id, redirectTombstone.RedirectToStackId); + Assert.DoesNotContain(RedirectedTag, redirectTombstone.Tags); + } + [Fact] public async Task RemoveTagsExceedingLimitsWhileKeepingKnownTags() { diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 17ffbe0a72..6ac9a60d2c 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -346,43 +346,65 @@ public async Task ReassignStackAsync_WithSourceEvents_MovesAllEventsToTarget() } [Fact] - public async Task RemoveAllByProjectIdsAsync_WithMixedEvents_RemovesOnlyMatchingEvents() + public async Task ReassignStackAsync_WithDuplicateMultipleSources_MovesEachEventOnce() { // Arrange - var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); - string otherProjectId = ObjectId.GenerateNewId().ToString(); - await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); - await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, otherProjectId, stack.Id), o => o.ImmediateConsistency()); + var source1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var source2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var target = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(4, TestConstants.OrganizationId, TestConstants.ProjectId, source1.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(6, TestConstants.OrganizationId, TestConstants.ProjectId, source2.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, TestConstants.ProjectId, target.Id), o => o.ImmediateConsistency()); // Act - long removed = await _repository.RemoveAllByProjectIdsAsync([TestConstants.ProjectId]); + long affected = await _repository.ReassignStackAsync( + [source1.Id, source1.Id, source2.Id], + target.Id, + TestContext.Current.CancellationToken); // Assert - Assert.Equal(10, removed); - + Assert.Equal(10, affected); await RefreshDataAsync(); - Assert.Equal(0, await _repository.CountAsync(q => q.Project(TestConstants.ProjectId))); - Assert.Equal(3, await _repository.CountAsync(q => q.Project(otherProjectId))); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(source1.Id))); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(source2.Id))); + Assert.Equal(12, await _repository.CountAsync(q => q.Stack(target.Id))); } [Fact] - public async Task RemoveAllByOrganizationIdsAsync_WithMixedEvents_RemovesOnlyMatchingEvents() + public async Task ReassignStackAsync_WithTargetInSources_ThrowsWithoutModification() { // Arrange - var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); - string otherOrganizationId = ObjectId.GenerateNewId().ToString(); - await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); - await _repository.AddAsync(_eventData.GenerateEvents(3, otherOrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + var source = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var target = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(4, TestConstants.OrganizationId, TestConstants.ProjectId, source.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, TestConstants.ProjectId, target.Id), o => o.ImmediateConsistency()); // Act - long removed = await _repository.RemoveAllByOrganizationIdsAsync([TestConstants.OrganizationId]); + await Assert.ThrowsAsync(() => + _repository.ReassignStackAsync([source.Id, target.Id], target.Id, TestContext.Current.CancellationToken)); // Assert - Assert.Equal(10, removed); + Assert.Equal(4, await _repository.CountAsync(q => q.Stack(source.Id))); + Assert.Equal(2, await _repository.CountAsync(q => q.Stack(target.Id))); + } - await RefreshDataAsync(); - Assert.Equal(0, await _repository.CountAsync(q => q.Organization(TestConstants.OrganizationId))); - Assert.Equal(3, await _repository.CountAsync(q => q.Organization(otherOrganizationId))); + [Fact] + public async Task ReassignStackAsync_WithCanceledToken_ThrowsWithoutModification() + { + // Arrange + var source = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var target = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(4, TestConstants.OrganizationId, TestConstants.ProjectId, source.Id), o => o.ImmediateConsistency()); + using var cancellationTokenSource = new CancellationTokenSource(); + await cancellationTokenSource.CancelAsync(); + + // Act + await Assert.ThrowsAnyAsync(() => + _repository.ReassignStackAsync([source.Id], target.Id, cancellationTokenSource.Token)); + + // Assert + Assert.Equal(4, await _repository.CountAsync(q => q.Stack(source.Id))); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(target.Id))); } [Fact] diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 8851c941f8..ab67ec2af7 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -181,7 +181,7 @@ public async Task CanIncrementEventCounterAsync() } [Fact] - public async Task SetEventCounterAsync_WhenIncomingValuesAreOlderOrLower_ShouldOnlyApplyMonotonicUpdates() + public async Task SetEventCounterAsync_WithNonMonotonicInputs_AppliesOnlyMonotonicUpdates() { // Arrange var originalFirst = new DateTime(2026, 2, 15, 0, 0, 0, DateTimeKind.Utc); @@ -226,6 +226,123 @@ await _repository.SetEventCounterAsync( Assert.Equal(originalLast.AddDays(1), updated.LastOccurrence); } + [Fact] + public async Task IncrementEventCounterAsync_WithReconciledRedirect_MarksReconciliationPending() + { + // Arrange + var source = await CreateReconciledRedirectAsync(); + + // Act + await _repository.IncrementEventCounterAsync( + source.OrganizationId, + source.ProjectId, + source.Id, + source.FirstOccurrence, + source.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false); + + // Assert + var updated = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(updated); + Assert.True(updated.NeedsRedirectReconciliation); + Assert.Equal(source.TotalOccurrences + 1, updated.TotalOccurrences); + } + + [Fact] + public async Task SetEventCounterAsync_WithReconciledRedirect_MarksReconciliationPending() + { + // Arrange + var source = await CreateReconciledRedirectAsync(); + + // Act + await _repository.SetEventCounterAsync( + source.Id, + source.FirstOccurrence.AddMinutes(-1), + source.LastOccurrence.AddMinutes(1), + source.TotalOccurrences + 1, + sendNotifications: false); + + // Assert + var updated = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(updated); + Assert.True(updated.NeedsRedirectReconciliation); + Assert.Equal(source.TotalOccurrences + 1, updated.TotalOccurrences); + } + + [Fact] + public async Task MarkDuplicateStackReconciledAsync_WithCounterUpdateAfterRead_LeavesReconciliationPending() + { + // Arrange + var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.TotalOccurrences = 10; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + var staleSource = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(staleSource); + + await _repository.IncrementEventCounterAsync( + source.OrganizationId, + source.ProjectId, + source.Id, + source.FirstOccurrence, + source.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false); + + // Act + await _repository.MarkDuplicateStackReconciledAsync(staleSource); + + // Assert + var updated = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(updated); + Assert.True(updated.NeedsRedirectReconciliation); + Assert.Equal(source.TotalOccurrences + 1, updated.TotalOccurrences); + } + + [Fact] + public async Task MarkDuplicateStackReconciledAsync_WithDateOnlyUpdateAtSameTimestamp_LeavesReconciliationPending() + { + // Arrange: keep updated_utc identical so the occurrence bounds must fence the write. + TimeProvider.SetUtcNow(new DateTime(2026, 8, 9, 12, 0, 0, DateTimeKind.Utc)); + var target = _stackData.GenerateStack( + generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + totalOccurrences: 10); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + var staleSource = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(staleSource); + DateTime earlierFirstOccurrence = staleSource.FirstOccurrence.AddMinutes(-1); + DateTime laterLastOccurrence = staleSource.LastOccurrence.AddMinutes(1); + + await _repository.SetEventCounterAsync( + source.Id, + earlierFirstOccurrence, + laterLastOccurrence, + staleSource.TotalOccurrences, + sendNotifications: false); + + // Act + await _repository.MarkDuplicateStackReconciledAsync(staleSource); + + // Assert + var updated = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(updated); + Assert.True(updated.NeedsRedirectReconciliation); + Assert.Equal(staleSource.TotalOccurrences, updated.TotalOccurrences); + Assert.Equal(earlierFirstOccurrence, updated.FirstOccurrence); + Assert.Equal(laterLastOccurrence, updated.LastOccurrence); + Assert.Equal(staleSource.UpdatedUtc, updated.UpdatedUtc); + } + [Fact] public async Task CanFindManyAsync() { @@ -356,19 +473,38 @@ public async Task GetSoftDeleted_WithRedirect_ExcludesRedirectTombstone() [Fact] public async Task GetCanonicalStackAsync_WithRedirect_ReturnsActiveTarget() { - var target = _stackData.GenerateSampleStack(); + // Arrange + var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); var source = target.DeepClone(); source.Id = ObjectId.GenerateNewId().ToString(); source.IsDeleted = true; source.RedirectToStackId = target.Id; await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + // Act var canonical = await _repository.GetCanonicalStackAsync(source.Id); - await _repository.AddEventTagsAsync(source.Id, ["redirected-tag"]); - var updatedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + // Assert Assert.NotNull(canonical); Assert.Equal(target.Id, canonical.Id); + } + + [Fact] + public async Task AddEventTagsAsync_WithRedirect_UpdatesCanonicalTarget() + { + // Arrange + var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.IsDeleted = true; + source.RedirectToStackId = target.Id; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + // Act + await _repository.AddEventTagsAsync(source.Id, ["redirected-tag"]); + var updatedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + + // Assert Assert.NotNull(updatedTarget); Assert.Contains("redirected-tag", updatedTarget.Tags); } @@ -574,6 +710,7 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc // Arrange var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); target.CreatedUtc = new DateTime(2026, 1, 2, 0, 0, 0, DateTimeKind.Utc); + target.FirstOccurrence = new DateTime(2026, 1, 2, 1, 0, 0, DateTimeKind.Utc); target.LastOccurrence = new DateTime(2026, 1, 2, 1, 0, 0, DateTimeKind.Utc); target.TotalOccurrences = 100; target.SnoozeUntilUtc = null; @@ -583,6 +720,7 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc var source = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); source.CreatedUtc = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + source.FirstOccurrence = new DateTime(2026, 1, 1, 1, 0, 0, DateTimeKind.Utc); source.LastOccurrence = new DateTime(2026, 1, 3, 1, 0, 0, DateTimeKind.Utc); source.TotalOccurrences = 10; source.Status = StackStatus.Fixed; @@ -618,6 +756,7 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc Assert.NotNull(merged); Assert.Equal(110, merged.TotalOccurrences); Assert.Equal(source.CreatedUtc, merged.CreatedUtc); + Assert.Equal(source.FirstOccurrence, merged.FirstOccurrence); Assert.Equal(source.LastOccurrence, merged.LastOccurrence); Assert.Equal(StackStatus.Fixed, merged.Status); Assert.Null(merged.SnoozeUntilUtc); @@ -639,4 +778,26 @@ public async Task MergeDuplicateStackAsync_WithRepeatedSource_AppliesMetadataOnc Assert.Equal(115, merged.TotalOccurrences); Assert.Equal(source.LastOccurrence, merged.LastOccurrence); } + + private async Task CreateReconciledRedirectAsync() + { + var target = _stackData.GenerateStack( + generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.TotalOccurrences = 10; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + source = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()) + ?? throw new InvalidOperationException(); + await _repository.MarkDuplicateStackReconciledAsync(source); + + source = await _repository.GetByIdAsync(source.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()) + ?? throw new InvalidOperationException(); + Assert.False(source.NeedsRedirectReconciliation); + return source; + } } diff --git a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs index aa6ffd787d..12428ad5a9 100644 --- a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs +++ b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs @@ -61,13 +61,14 @@ public void DataDictionaryExtensions_GetValue_PreservesJsonSerializerOptionsOver } [Fact] - public void Stack_InternalCleanupState_IsNotSerializedByApi() + public void ConfigureExceptionlessApiDefaults_StackWithInternalState_OmitsInternalProperties() { var stack = new Stack { RedirectToStackId = "target-stack", MergedDuplicateStackTotals = new Dictionary { ["source-stack"] = 42 }, - NeedsRedirectReconciliation = true + NeedsRedirectReconciliation = true, + ElasticVersion = "42" }; string apiJson = JsonSerializer.Serialize(stack, new JsonSerializerOptions().ConfigureExceptionlessApiDefaults()); @@ -76,8 +77,10 @@ public void Stack_InternalCleanupState_IsNotSerializedByApi() Assert.DoesNotContain("redirect_to_stack_id", apiJson); Assert.DoesNotContain("merged_duplicate_stack_totals", apiJson); Assert.DoesNotContain("needs_redirect_reconciliation", apiJson); + Assert.DoesNotContain("elastic_version", apiJson); Assert.Contains("redirect_to_stack_id", storageJson); Assert.Contains("merged_duplicate_stack_totals", storageJson); Assert.Contains("needs_redirect_reconciliation", storageJson); + Assert.Contains("elastic_version", storageJson); } } From 253dca8e5912d0e65f7124b646af32bd23fc7a54 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 11 Aug 2026 13:58:40 -0500 Subject: [PATCH 6/9] fix: fail closed on parent read errors --- .../Jobs/CleanupOrphanedDataJob.cs | 27 +++-- .../Repositories/Base/RepositoryBase.cs | 88 +++++++++++++- .../StrictRepositoryExtensions.cs | 23 ++++ .../Jobs/CleanupOrphanedDataJobSafetyTests.cs | 112 ++++++++++++++++++ 4 files changed, 241 insertions(+), 9 deletions(-) create mode 100644 src/Exceptionless.Core/Repositories/StrictRepositoryExtensions.cs create mode 100644 tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index d16a9ed089..e1e9f5abbf 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -82,10 +82,14 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) hasMore = !String.IsNullOrEmpty(nextValue); totalStackIds += stackIds.Count; - // Keep this destructive existence check on real-time multi-get. An OnlyIds search can - // continue with stale results after a failed refresh, and IsDeleted must be hydrated - // because default soft-delete filtering is applied client-side to multi-get results. - var existingStacks = await _stackRepository.GetByIdsAsync(stackIds.ToArray(), o => o.Include(s => s.Id, s => s.IsDeleted)); + // Keep this destructive existence check on strict real-time multi-get. An OnlyIds + // search can continue with stale results after a failed refresh, and per-item read + // errors must never look like missing parents. IsDeleted must also be hydrated because + // default soft-delete filtering is applied client-side to multi-get results. + var existingStacks = await _stackRepository.GetByIdsStrictAsync( + stackIds.ToArray(), + o => o.Include(s => s.Id, s => s.IsDeleted), + context.CancellationToken); var existingStackIds = existingStacks.Select(s => s.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingStackIds = stackIds.Where(id => !existingStackIds.Contains(id)).ToArray(); @@ -95,9 +99,10 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) // Redirect tombstones are intentionally retained so events from an in-flight ingestion // context can never be mistaken for orphaned data. Move those late events to the // canonical stack and refresh its metadata before deleting only truly missing stacks. - var redirectedStacks = await _stackRepository.GetByIdsAsync( + var redirectedStacks = await _stackRepository.GetByIdsStrictAsync( missingStackIds, - o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + o => o.SoftDeleteMode(SoftDeleteQueryMode.All), + context.CancellationToken); var redirectedStackIds = redirectedStacks .Where(s => !String.IsNullOrEmpty(s.RedirectToStackId)) .Select(s => s.Id) @@ -147,7 +152,10 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) hasMore = !String.IsNullOrEmpty(nextValue); totalProjectIds += projectIds.Count; - var existingProjects = await _projectRepository.GetByIdsAsync(projectIds.ToArray(), o => o.Include(p => p.Id, p => p.IsDeleted)); + var existingProjects = await _projectRepository.GetByIdsStrictAsync( + projectIds.ToArray(), + o => o.Include(p => p.Id, p => p.IsDeleted), + context.CancellationToken); var existingProjectIds = existingProjects.Select(p => p.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingProjectIds = projectIds.Where(id => !existingProjectIds.Contains(id)).ToArray(); @@ -186,7 +194,10 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) hasMore = !String.IsNullOrEmpty(nextValue); totalOrganizationIds += organizationIds.Count; - var existingOrganizations = await _organizationRepository.GetByIdsAsync(organizationIds.ToArray(), o => o.Include(organization => organization.Id, organization => organization.IsDeleted)); + var existingOrganizations = await _organizationRepository.GetByIdsStrictAsync( + organizationIds.ToArray(), + o => o.Include(organization => organization.Id, organization => organization.IsDeleted), + context.CancellationToken); var existingOrganizationIds = existingOrganizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); string[] missingOrganizationIds = organizationIds.Where(id => !existingOrganizationIds.Contains(id)).ToArray(); diff --git a/src/Exceptionless.Core/Repositories/Base/RepositoryBase.cs b/src/Exceptionless.Core/Repositories/Base/RepositoryBase.cs index a506980d07..b1132c4697 100644 --- a/src/Exceptionless.Core/Repositories/Base/RepositoryBase.cs +++ b/src/Exceptionless.Core/Repositories/Base/RepositoryBase.cs @@ -1,10 +1,15 @@ -using Exceptionless.Core.Messaging.Models; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.MGet; +using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Options; using Exceptionless.Core.Validation; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Exceptions; +using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Foundatio.Repositories.Queries; @@ -23,6 +28,87 @@ public RepositoryBase(IIndex index, MiniValidationValidator validator, AppOption NotificationsEnabled = options.EnableRepositoryNotifications; } + /// + /// Gets documents in real time and rejects top-level or per-document multi-get errors. + /// Use this when a missing result can trigger a destructive operation: Elasticsearch can + /// return HTTP 200 with an error for an individual item, which the standard repository + /// multi-get intentionally logs and omits from its result set. This bypasses cache reads so + /// the result reflects Elasticsearch's real-time view. + /// + internal async Task> GetByIdsStrictAsync( + Foundatio.Repositories.Ids ids, + CommandOptionsDescriptor? options = null, + CancellationToken cancellationToken = default) + { + var idList = ids?.Distinct().Where(id => !String.IsNullOrEmpty(id)).ToList(); + if (idList is not { Count: > 0 }) + return []; + if (HasParent || ElasticIndex.HasMultipleIndexes) + throw new NotSupportedException("Strict multi-get only supports single-index repositories without parent routing."); + + var remainingIds = idList.Select(id => id.Value).ToHashSet(StringComparer.Ordinal); + if (remainingIds.Count != idList.Count) + throw new NotSupportedException("Strict multi-get does not support duplicate IDs with different routing values."); + + var configuredOptions = ConfigureOptions(options.Configure()); + await OnBeforeGetAsync(new Foundatio.Repositories.Ids(idList), configuredOptions, typeof(T)); + var operations = idList.Select(id => + { + var operation = new MultiGetOperation(id.Value) { Index = ElasticIndex.GetIndex(id) }; + if (id.Routing is not null) + operation.Routing = id.Routing; + + return operation; + }).ToList(); + + var request = new MultiGetRequestDescriptor().Docs(operations); + ConfigureMultiGetRequest(request, configuredOptions); + + var response = await _client.MultiGetAsync(request, cancellationToken); + _logger.LogRequest(response, configuredOptions.GetQueryLogLevel()); + if (!response.IsValidResponse) + throw new DocumentException($"Error getting documents: {response.DebugInformation}", response.ApiCallDetails.OriginalException); + + var documents = new List(); + foreach (var item in response.Docs) + { + item.Match( + result => + { + if (result is null || String.IsNullOrEmpty(result.Id) || !remainingIds.Remove(result.Id)) + throw new DocumentException("Elasticsearch returned an invalid multi-get item."); + + if (result.Found) + { + if (result.Source is null || !String.Equals(result.Source.Id, result.Id, StringComparison.Ordinal)) + throw new DocumentException($"Elasticsearch returned document {result.Id} without a matching source."); + + if (result.Source is IVersioned versionedDocument && result.PrimaryTerm.HasValue && result.SeqNo.HasValue) + versionedDocument.Version = new ElasticDocumentVersion(result.PrimaryTerm.Value, result.SeqNo.Value); + + if (ShouldReturnDocument(result.Source, configuredOptions)) + documents.Add(result.Source); + } + else if (result.Source is not null) + { + throw new DocumentException($"Elasticsearch returned a source for missing document {result.Id}."); + } + }, + error => + { + if (error is null) + throw new DocumentException("Elasticsearch returned an invalid multi-get error item."); + + throw new DocumentException($"Error getting document {error.Id} from index {error.Index}: {error.Error?.Reason}"); + }); + } + + if (remainingIds.Count > 0) + throw new DocumentException($"Elasticsearch omitted {remainingIds.Count} requested multi-get item(s)."); + + return documents.AsReadOnly(); + } + protected override Task ValidateAndThrowAsync(T document) { return _validator.ValidateAndThrowAsync(document); diff --git a/src/Exceptionless.Core/Repositories/StrictRepositoryExtensions.cs b/src/Exceptionless.Core/Repositories/StrictRepositoryExtensions.cs new file mode 100644 index 0000000000..0cf0613a40 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/StrictRepositoryExtensions.cs @@ -0,0 +1,23 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Repositories; + +internal static class StrictRepositoryExtensions +{ + public static Task> GetByIdsStrictAsync( + this IReadOnlyRepository repository, + Ids ids, + CommandOptionsDescriptor? options = null, + CancellationToken cancellationToken = default) + where T : class, IIdentity, new() + { + // The cleanup job's production repositories are all RepositoryBase implementations. + // Unsupported substitutes must fail closed instead of treating unreadable parents as absent. + if (repository is not RepositoryBase exceptionlessRepository) + throw new NotSupportedException($"Repository {repository.GetType().Name} does not support strict multi-get reads."); + + return exceptionlessRepository.GetByIdsStrictAsync(ids, options, cancellationToken); + } +} diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs new file mode 100644 index 0000000000..d8d23fae89 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs @@ -0,0 +1,112 @@ +using Exceptionless.Core.Billing; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Repositories; +using Foundatio.Repositories.Exceptions; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class CleanupOrphanedDataJobSafetyTests : IntegrationTestsBase +{ + private readonly CleanupOrphanedDataJob _job; + private readonly OrganizationData _organizationData; + private readonly IOrganizationRepository _organizationRepository; + private readonly ProjectData _projectData; + private readonly IProjectRepository _projectRepository; + private readonly StackData _stackData; + private readonly IStackRepository _stackRepository; + private readonly EventData _eventData; + private readonly IEventRepository _eventRepository; + private readonly BillingManager _billingManager; + private readonly BillingPlans _plans; + private readonly ExceptionlessElasticConfiguration _configuration; + + public CleanupOrphanedDataJobSafetyTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _job = GetService(); + _organizationData = GetService(); + _organizationRepository = GetService(); + _projectData = GetService(); + _projectRepository = GetService(); + _stackData = GetService(); + _stackRepository = GetService(); + _eventData = GetService(); + _eventRepository = GetService(); + _billingManager = GetService(); + _plans = GetService(); + _configuration = GetService(); + } + + [Fact] + public async Task DeleteOrphanedEventsByStackAsync_WithUnavailableStackIndex_FailsWithoutDeletingEvents() + { + string stackId = await CreateValidEventAsync(); + + await AssertUnavailableParentIndexPreservesEventsAsync( + _configuration.Stacks.VersionedName, + context => _job.DeleteOrphanedEventsByStackAsync(context), + stackId); + } + + [Fact] + public async Task DeleteOrphanedEventsByProjectAsync_WithUnavailableProjectIndex_FailsWithoutDeletingEvents() + { + string stackId = await CreateValidEventAsync(); + + await AssertUnavailableParentIndexPreservesEventsAsync( + _configuration.Projects.VersionedName, + context => _job.DeleteOrphanedEventsByProjectAsync(context), + stackId); + } + + [Fact] + public async Task DeleteOrphanedEventsByOrganizationAsync_WithUnavailableOrganizationIndex_FailsWithoutDeletingEvents() + { + string stackId = await CreateValidEventAsync(); + + await AssertUnavailableParentIndexPreservesEventsAsync( + _configuration.Organizations.VersionedName, + context => _job.DeleteOrphanedEventsByOrganizationAsync(context), + stackId); + } + + private async Task CreateValidEventAsync() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvent(organization.Id, project.Id, stack.Id), + o => o.ImmediateConsistency()); + + return stack.Id; + } + + private async Task AssertUnavailableParentIndexPreservesEventsAsync( + string parentIndex, + Func cleanup, + string stackId) + { + var closeResponse = await _configuration.Client.Indices.CloseAsync(parentIndex, TestContext.Current.CancellationToken); + Assert.True(closeResponse.IsValidResponse, closeResponse.DebugInformation); + + try + { + var exception = await Assert.ThrowsAsync(() => cleanup(new JobContext(TestCancellationToken))); + Assert.StartsWith("Error getting document ", exception.Message); + } + finally + { + var openResponse = await _configuration.Client.Indices.OpenAsync(parentIndex, TestContext.Current.CancellationToken); + Assert.True(openResponse.IsValidResponse, openResponse.DebugInformation); + } + + await RefreshDataAsync(); + Assert.Equal(1, await _eventRepository.CountAsync(q => q.Stack(stackId), o => o.ImmediateConsistency())); + } +} From ba1aca0b547689b9e7a7fbddefb010c6c7cab9e7 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 11 Aug 2026 14:04:24 -0500 Subject: [PATCH 7/9] fix: preserve concurrently restored stacks --- .../Jobs/CleanupOrphanedDataJob.cs | 21 +++++++---- .../Jobs/CleanupOrphanedDataJobSafetyTests.cs | 37 +++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index e1e9f5abbf..b491e0f606 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -96,25 +96,32 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) if (missingStackIds.Length is 0) continue; - // Redirect tombstones are intentionally retained so events from an in-flight ingestion - // context can never be mistaken for orphaned data. Move those late events to the - // canonical stack and refresh its metadata before deleting only truly missing stacks. - var redirectedStacks = await _stackRepository.GetByIdsStrictAsync( + // Recheck missing IDs with soft deletes included: a concurrently restored active stack + // is still valid, and redirect tombstones are intentionally retained so events from an + // in-flight ingestion context can never be mistaken for orphaned data. Move late events + // to the canonical stack before deleting only truly missing or deleted stacks. + var recheckedStacks = await _stackRepository.GetByIdsStrictAsync( missingStackIds, o => o.SoftDeleteMode(SoftDeleteQueryMode.All), context.CancellationToken); - var redirectedStackIds = redirectedStacks + var activeRecheckedStackIds = recheckedStacks + .Where(s => !s.IsDeleted) + .Select(s => s.Id) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var redirectedStackIds = recheckedStacks .Where(s => !String.IsNullOrEmpty(s.RedirectToStackId)) .Select(s => s.Id) .ToHashSet(StringComparer.OrdinalIgnoreCase); - var redirectedStackDocuments = redirectedStacks + var redirectedStackDocuments = recheckedStacks .Where(s => redirectedStackIds.Contains(s.Id)) .ToList(); if (redirectedStackDocuments.Count > 0) await ReconcileRedirectedStacksAsync(redirectedStackDocuments, context); - string[] orphanedStackIds = missingStackIds.Where(id => !redirectedStackIds.Contains(id)).ToArray(); + string[] orphanedStackIds = missingStackIds + .Where(id => !activeRecheckedStackIds.Contains(id) && !redirectedStackIds.Contains(id)) + .ToArray(); if (orphanedStackIds.Length is 0) continue; diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs index d8d23fae89..9f0f1fd7f5 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobSafetyTests.cs @@ -6,6 +6,7 @@ using Foundatio.Jobs; using Foundatio.Repositories; using Foundatio.Repositories.Exceptions; +using Foundatio.Repositories.Options; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -74,6 +75,42 @@ await AssertUnavailableParentIndexPreservesEventsAsync( stackId); } + [Fact] + public async Task DeleteOrphanedEventsByStackAsync_WithStackRestoredBetweenExistenceChecks_PreservesEvents() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = _stackData.GenerateSampleStack(); + stack.IsDeleted = true; + await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvent(organization.Id, project.Id, stack.Id), + o => o.ImmediateConsistency()); + + bool restoredDuringRecheck = false; + var stackRepository = Assert.IsType(_stackRepository); + using var handler = stackRepository.BeforeGet.AddHandler(async (_, args) => + { + if (restoredDuringRecheck + || args.Options.GetSoftDeleteMode() is not SoftDeleteQueryMode.All + || !args.Ids.Any(id => String.Equals(id.Value, stack.Id, StringComparison.Ordinal))) + { + return; + } + + restoredDuringRecheck = true; + stack.IsDeleted = false; + await _stackRepository.SaveAsync(stack, o => o.ImmediateConsistency()); + }); + + await _job.DeleteOrphanedEventsByStackAsync(new JobContext(TestCancellationToken)); + + Assert.True(restoredDuringRecheck); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency())); + Assert.Equal(1, await _eventRepository.CountAsync(q => q.Stack(stack.Id), o => o.ImmediateConsistency())); + } + private async Task CreateValidEventAsync() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); From 51465b9ee0e30e9b69444e31ffe17c14cd2e9f5b Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:35:02 -0500 Subject: [PATCH 8/9] Revert "Merge remote-tracking branch 'origin/main' into feature/pr-2267-live-0700" This reverts commit ad119ccfaad5016407a45bb6744da395f0dc52e4, reversing changes made to 283d7ee892089bcbf2722581a5d58b9dcdcae0dd. --- .../e2e/tests/stack-effects-chaos.e2e.ts | 25 ------ .../src/lib/features/auth/api.svelte.ts | 11 ++- .../src/lib/features/auth/api.test.ts | 5 -- .../src/lib/features/auth/index.svelte.ts | 15 +++- .../src/lib/features/auth/session.svelte.ts | 8 -- .../src/lib/features/auth/state.svelte.ts | 12 --- .../lib/features/auth/unauthorized.test.ts | 29 ------- .../src/lib/features/auth/unauthorized.ts | 11 --- .../components/events-dashboard-chart.svelte | 2 + .../intercom/intercom-initializer.svelte | 66 +++++++-------- .../intercom/intercom-shell.svelte.test.ts | 84 ++++--------------- .../intercom-shell.test-harness.svelte | 5 +- .../features/intercom/session.svelte.test.ts | 39 --------- .../src/lib/features/intercom/session.ts | 47 ----------- .../src/lib/features/intercom/updates.test.ts | 79 ----------------- .../src/lib/features/intercom/updates.ts | 57 ------------- .../sessions-dashboard-chart.svelte | 2 + .../shared/components/object-dump.svelte | 2 +- .../components/object-dump.svelte.test.ts | 15 ---- .../components/ui/chart/chart-tooltip.svelte | 2 +- .../ClientApp/src/routes/(app)/+layout.svelte | 3 +- .../(auth)/oauth/authorize/+page.svelte | 3 +- .../ClientApp/src/routes/+layout.svelte | 7 +- src/Exceptionless.Web/Program.cs | 12 +-- 24 files changed, 85 insertions(+), 456 deletions(-) delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts index 3f8a565737..6a99af1d4f 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts @@ -91,31 +91,6 @@ test('stack effects stay bounded through background, paging, and navigation chao await expect(page.getByRole('heading', { name: 'Stacks' })).toBeVisible(); }); - await measureAction(diagnostics, 'stack timeline drag', async () => { - await page.locator('tbody tr:visible').first().click(); - const dialog = page.getByRole('dialog'); - await expect(dialog).toBeVisible(); - const timeline = dialog.locator('[data-slot="chart"]'); - await expect(timeline).toBeVisible(); - - const bounds = await timeline.boundingBox(); - expect(bounds).not.toBeNull(); - if (bounds) { - const y = bounds.y + bounds.height / 2; - await page.mouse.move(bounds.x + bounds.width * 0.2, y); - await page.waitForTimeout(250); - await page.mouse.move(bounds.x + bounds.width * 0.3, y); - await expect(page.locator('.lc-tooltip-root')).toHaveCSS('pointer-events', 'none'); - await page.mouse.down(); - await page.mouse.move(bounds.x + bounds.width * 0.8, y, { steps: 20 }); - await page.mouse.up(); - } - - await page.waitForTimeout(500); - await page.getByRole('button', { name: 'Close' }).click(); - }); - expect(actionSample(diagnostics, 'stack timeline drag').runtimeErrors).toBe(0); - await measureAction(diagnostics, 'selected stack refresh', async () => { const rowSelection = page.getByRole('checkbox', { name: 'Select row' }).first(); await rowSelection.click(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts index ec9ed03098..306ecf376c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts @@ -2,13 +2,13 @@ import { env } from '$env/dynamic/public'; import { getIntercomTokenSessionKey, intercomTokenRefreshIntervalMs } from '$features/intercom/config'; import { organization } from '$features/organizations/context.svelte'; import { ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; +import { hide as hideIntercom, shutdown as shutdownIntercom } from '@intercom/messenger-js-sdk'; import { createQuery, type QueryClient } from '@tanstack/svelte-query'; import type { Login, TokenResult } from './models'; import { endSession } from './exceptionless-session'; -import { clearAuthenticationSession } from './session.svelte'; -import { accessToken } from './state.svelte'; +import { accessToken } from './index.svelte'; const queryKeys = { intercom: (accessToken: null | string) => ['Auth', 'intercom', getIntercomTokenSessionKey(accessToken)] as const @@ -103,12 +103,17 @@ export async function logout(queryClient?: QueryClient, client = useFetchClient( await queryClient?.cancelQueries(); queryClient?.clear(); - clearAuthenticationSession(); + if (typeof window !== 'undefined' && 'Intercom' in window && typeof window.Intercom === 'function') { + hideIntercom(); + shutdownIntercom(); + } organization.current = undefined; if (typeof localStorage !== 'undefined') { localStorage.removeItem('organization'); } + + accessToken.current = null; } export async function resetPassword(passwordResetToken: string, password: string) { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts index 729a0a4dae..6be26032b9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts @@ -1,18 +1,14 @@ import { FetchClient } from '@foundatiofx/fetchclient'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -const clearAuthenticationSession = vi.hoisted(() => vi.fn()); - vi.mock('./exceptionless-session', () => ({ endSession: vi.fn() })); -vi.mock('./session.svelte', () => ({ clearAuthenticationSession })); import { logout } from './api.svelte'; describe('logout', () => { beforeEach(() => { - clearAuthenticationSession.mockReset(); // Mock localStorage for server-side tests Object.defineProperty(globalThis, 'localStorage', { configurable: true, @@ -32,6 +28,5 @@ describe('logout', () => { await logout(undefined, mockClient); expect(mockClient.get).toHaveBeenCalledWith('auth/logout', { expectedStatusCodes: [200, 401, 403] }); - expect(clearAuthenticationSession).toHaveBeenCalledOnce(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts index 0581a25cbb..b0a247a3ba 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts @@ -2,12 +2,11 @@ import { goto } from '$app/navigation'; import { resolve } from '$app/paths'; import { page } from '$app/state'; import { env } from '$env/dynamic/public'; +import { CachedPersistedState } from '$features/shared/utils/cached-persisted-state.svelte'; import { useFetchClient } from '@foundatiofx/fetchclient'; import type { TokenResult } from './models'; -import { accessToken } from './state.svelte'; - // Re-export all API functions for backward compatibility export { cancelResetPassword, @@ -22,7 +21,6 @@ export { unlinkOAuthAccount } from './api.svelte'; -export { accessToken } from './state.svelte'; // Re-export validators export { validateEmailAvailability } from './validators'; @@ -46,6 +44,17 @@ export interface OAuthResponseData { export type SupportedOAuthProviders = 'facebook' | 'github' | 'google' | 'live' | 'slack'; +const authSerializer = { + deserialize: (value: null | string): null | string => { + return value === '' ? null : value; + }, + serialize: (value: null | string): string => { + return value === null ? '' : value; + } +}; + +export const accessToken = new CachedPersistedState('satellizer_token', null, { serializer: authSerializer }); + export const enableAccountCreation = env.PUBLIC_ENABLE_ACCOUNT_CREATION === 'true'; export const facebookClientId = env.PUBLIC_FACEBOOK_APPID; export const gitHubClientId = env.PUBLIC_GITHUB_APPID; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts deleted file mode 100644 index 1882ade1fa..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { shutdownIntercomSession } from '$features/intercom/session'; - -import { accessToken } from './state.svelte'; - -export function clearAuthenticationSession(clearedAccessToken: '' | null = null) { - shutdownIntercomSession(); - accessToken.current = clearedAccessToken; -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts deleted file mode 100644 index ef9a1973dd..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { CachedPersistedState } from '$features/shared/utils/cached-persisted-state.svelte'; - -const authSerializer = { - deserialize: (value: null | string): null | string => { - return value === '' ? null : value; - }, - serialize: (value: null | string): string => { - return value === null ? '' : value; - } -}; - -export const accessToken = new CachedPersistedState('satellizer_token', null, { serializer: authSerializer }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts deleted file mode 100644 index 461f62c168..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -const clearAuthenticationSession = vi.hoisted(() => vi.fn()); -const accessToken = vi.hoisted(() => ({ current: 'token_123' as null | string })); - -vi.mock('./session.svelte', () => ({ clearAuthenticationSession })); -vi.mock('./state.svelte', () => ({ accessToken })); - -import { handleUnexpectedUnauthorized } from './unauthorized'; - -describe('handleUnexpectedUnauthorized', () => { - beforeEach(() => { - accessToken.current = 'token_123'; - clearAuthenticationSession.mockReset(); - }); - - it('clears the authenticated session after an unexpected unauthorized response', () => { - expect(handleUnexpectedUnauthorized(401)).toBe(true); - expect(clearAuthenticationSession).toHaveBeenCalledExactlyOnceWith(''); - }); - - it('ignores expected unauthorized responses and duplicate session cleanup', () => { - expect(handleUnexpectedUnauthorized(401, [401])).toBe(false); - - accessToken.current = null; - expect(handleUnexpectedUnauthorized(401)).toBe(false); - expect(clearAuthenticationSession).not.toHaveBeenCalled(); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts deleted file mode 100644 index 7174a30448..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { clearAuthenticationSession } from './session.svelte'; -import { accessToken } from './state.svelte'; - -export function handleUnexpectedUnauthorized(status: number, expectedStatusCodes?: number[]) { - if (status !== 401 || expectedStatusCodes?.includes(401) || !accessToken.current) { - return false; - } - - clearAuthenticationSession(''); - return true; -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte index 869049ccb7..7be0e2b734 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte @@ -72,6 +72,8 @@ if (start instanceof Date && end instanceof Date) { onRangeSelect?.(start, end); } + + e.brush.reset(); } }} props={{ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte index 5803befbd5..036b98951e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte @@ -8,68 +8,66 @@ import type { Snippet } from 'svelte'; import type { BootOptions } from 'svelte-intercom'; + import { accessToken } from '$features/auth/index.svelte'; + import { DocumentVisibility } from '$shared/document-visibility.svelte'; + import { useInterval } from 'runed'; import { untrack } from 'svelte'; import { setContext } from 'svelte'; import { useIntercom } from 'svelte-intercom'; import { INTERCOM_CONTEXT_KEY } from './keys'; - import { buildIntercomDataUpdate, buildIntercomRouteUpdate } from './updates'; interface Props { bootOptions?: BootOptions; children: Snippet; routeKey?: string; - /** @deprecated Intercom updates are event-driven; this value is retained as a no-op for compatibility. */ updateIntervalMs?: number; } - let { bootOptions = undefined, children, routeKey = undefined, updateIntervalMs = undefined }: Props = $props(); + let { bootOptions = undefined, children, routeKey = undefined, updateIntervalMs = 90_000 }: Props = $props(); const intercom = useIntercom(); - let hasBooted = false; - let previousBootOptions: BootOptions | undefined; - let previousRouteKey: string | undefined; + const visibility = new DocumentVisibility(); setContext(INTERCOM_CONTEXT_KEY, intercom); - // Retain the deprecated prop reactively without restoring periodic updates. - $effect(() => { - void updateIntervalMs; + const interval = useInterval(() => updateIntervalMs, { + callback: () => { + if (bootOptions && visibility.visible) { + intercom.update(bootOptions); + } + }, + immediate: false }); - // The provider boots with the initial options. Only update after boot when the route or - // identity/company data changes; eager or periodic updates create duplicate impressions. + const shouldUpdate = $derived(bootOptions && visibility.visible); + + // Sync identity/company data and manage interval when boot options or visibility changes. $effect(() => { - const options = bootOptions; - const currentRouteKey = routeKey; - if (!options) { - hasBooted = false; - previousBootOptions = undefined; - previousRouteKey = undefined; + if (!bootOptions) { + interval.pause(); return; } - if (!hasBooted) { - hasBooted = true; - previousBootOptions = options; - previousRouteKey = currentRouteKey; - return; + if (visibility.visible) { + interval.resume(); + } else { + interval.pause(); } + }); - if (typeof window.Intercom !== 'function') { - return; + // Sync on route transitions and visibility changes. + $effect(() => { + void routeKey; + if (shouldUpdate) { + untrack(() => intercom.update(bootOptions!)); } + }); - const priorBootOptions = previousBootOptions!; - const bootOptionsChanged = options !== priorBootOptions; - const routeChanged = currentRouteKey !== previousRouteKey; - previousBootOptions = options; - previousRouteKey = currentRouteKey; - - if (bootOptionsChanged) { - untrack(() => intercom.update(buildIntercomDataUpdate(priorBootOptions, options))); - } else if (routeChanged) { - untrack(() => intercom.update(buildIntercomRouteUpdate(options))); + // Shutdown when the user logs out. + $effect(() => { + if (!accessToken.current) { + untrack(() => intercom.shutdown()); } }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts index a09ab0e06d..dbeb4f03ea 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts @@ -43,7 +43,6 @@ describe('IntercomShell', () => { intercomShowMessages.mockReset(); intercomUpdate.mockReset(); vi.restoreAllMocks(); - window.Intercom = vi.fn(); }); it('keeps children mounted when Intercom becomes bootable', async () => { @@ -77,84 +76,37 @@ describe('IntercomShell', () => { expect(intercomShowMessages).toHaveBeenCalledTimes(1); }); - it('updates after boot only when the route or boot options change', async () => { + it('remains stable across repeated tab visibility changes', async () => { // Arrange - const bootOptions = { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions; + let hidden = false; + const addEventListener = vi.spyOn(document, 'addEventListener'); + vi.spyOn(document, 'hidden', 'get').mockImplementation(() => hidden); const { rerender } = render(IntercomShellTestHarness, { props: { appId: 'app_123', - bootOptions, - routeKey: '/event/all' + bootOptions: { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions } }); await tick(); - // Assert initial boot options are not immediately sent again. - expect(intercomUpdate).not.toHaveBeenCalled(); - - // Act - await rerender({ appId: 'app_123', bootOptions, routeKey: '/stack/all' }); - - // Assert - expect(intercomUpdate).toHaveBeenCalledOnce(); - expect(intercomUpdate).toHaveBeenLastCalledWith({ - intercom_user_jwt: 'token_0', - last_request_at: expect.any(Number), - user_id: 'user_123' - }); - // Act - await rerender({ - appId: 'app_123', - bootOptions: { intercomUserJwt: 'token_1', userId: 'user_123' } as BootOptions, - routeKey: '/stack/all' - }); - - // Assert - expect(intercomUpdate).toHaveBeenCalledTimes(2); - expect(intercomUpdate).toHaveBeenLastCalledWith({ - intercom_user_jwt: 'token_1', - user_id: 'user_123' - }); - }); + for (let index = 0; index < 100; index++) { + hidden = true; + document.dispatchEvent(new Event('visibilitychange')); + await tick(); - it('does not update when navigation stays within the same normalized route', async () => { - const bootOptions = { email: 'user@example.com', userId: 'user_123' } as BootOptions; - const routeKey = '/(app)/project/[projectId]/event/[eventId]'; - const { rerender } = render(IntercomShellTestHarness, { - props: { appId: 'app_123', bootOptions, routeKey } - }); - await tick(); - - await rerender({ appId: 'app_123', bootOptions, routeKey }); - - expect(intercomUpdate).not.toHaveBeenCalled(); - }); - - it('does not update before the client SDK initializes', async () => { - // Arrange - window.Intercom = undefined; - const bootOptions = { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions; - const { rerender } = render(IntercomShellTestHarness, { - props: { + await rerender({ appId: 'app_123', - bootOptions, - routeKey: '/event/all' - } - }); - await tick(); - - // Act - await rerender({ appId: 'app_123', bootOptions, routeKey: '/stack/all' }); + bootOptions: { intercomUserJwt: `token_${index + 1}`, userId: 'user_123' } as BootOptions + }); - // Assert - expect(intercomUpdate).not.toHaveBeenCalled(); - - // Act - window.Intercom = vi.fn(); - await rerender({ appId: 'app_123', bootOptions, routeKey: '/event/errors' }); + hidden = false; + document.dispatchEvent(new Event('visibilitychange')); + await tick(); + } // Assert - expect(intercomUpdate).toHaveBeenCalledOnce(); + expect(intercomUpdate).toHaveBeenCalled(); + expect(addEventListener.mock.calls.filter(([eventName]) => eventName === 'visibilitychange')).toHaveLength(1); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte index 5e1655bc38..8ff9d77876 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte @@ -8,13 +8,12 @@ appId?: string; bootOptions?: BootOptions; onMountProbe?: () => void; - routeKey?: string; } - let { appId = undefined, bootOptions = undefined, onMountProbe = () => {}, routeKey = undefined }: Props = $props(); + let { appId = undefined, bootOptions = undefined, onMountProbe = () => {} }: Props = $props(); - + {#snippet children(openChat)} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts deleted file mode 100644 index 113ad71707..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -const hide = vi.hoisted(() => vi.fn()); -const shutdown = vi.hoisted(() => vi.fn()); - -vi.mock('@intercom/messenger-js-sdk', () => ({ hide, shutdown })); - -import { shutdownIntercomSession } from './session'; - -describe('shutdownIntercomSession', () => { - beforeEach(() => { - hide.mockReset(); - shutdown.mockReset(); - window.Intercom = vi.fn(); - document.cookie = 'intercom-id-app=visitor; Path=/'; - document.cookie = 'intercom-session-app=session; Path=/'; - document.cookie = 'unrelated-cookie=keep; Path=/'; - }); - - it('shuts down the SDK and clears all Intercom cookies', () => { - shutdownIntercomSession(); - - expect(hide).toHaveBeenCalledOnce(); - expect(shutdown).toHaveBeenCalledOnce(); - expect(document.cookie).not.toContain('intercom-id-app'); - expect(document.cookie).not.toContain('intercom-session-app'); - expect(document.cookie).toContain('unrelated-cookie=keep'); - }); - - it('still clears cookies when tracking prevention blocks the SDK', () => { - window.Intercom = undefined; - - expect(() => shutdownIntercomSession()).not.toThrow(); - expect(hide).not.toHaveBeenCalled(); - expect(shutdown).not.toHaveBeenCalled(); - expect(document.cookie).not.toContain('intercom-id-app'); - expect(document.cookie).not.toContain('intercom-session-app'); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts deleted file mode 100644 index 2c5f7082ad..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { hide, shutdown } from '@intercom/messenger-js-sdk'; - -export function clearIntercomCookies() { - if (typeof document === 'undefined') { - return; - } - - const cookieNames = new Set( - document.cookie - .split(';') - .map((cookie) => cookie.trim().split('=', 1)[0] ?? '') - .filter((name) => name.startsWith('intercom-')) - ); - - const hostname = typeof window === 'undefined' ? '' : window.location.hostname; - const domainCandidates = getCookieDomainCandidates(hostname); - - for (const name of cookieNames) { - expireCookie(name); - for (const domain of domainCandidates) { - expireCookie(name, domain); - } - } -} - -export function shutdownIntercomSession() { - if (typeof window !== 'undefined' && typeof window.Intercom === 'function') { - hide(); - shutdown(); - } - - clearIntercomCookies(); -} - -function expireCookie(name: string, domain?: string) { - const domainAttribute = domain ? `; Domain=${domain}` : ''; - document.cookie = `${name}=; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/${domainAttribute}; SameSite=Lax`; -} - -function getCookieDomainCandidates(hostname: string) { - if (!hostname.includes('.') || /^\d{1,3}(\.\d{1,3}){3}$/.test(hostname)) { - return []; - } - - const parts = hostname.split('.'); - return parts.slice(0, -1).map((_, index) => parts.slice(index).join('.')); -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts deleted file mode 100644 index 646684f360..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import type { BootOptions } from 'svelte-intercom'; - -import { describe, expect, it } from 'vitest'; - -import { buildIntercomDataUpdate, buildIntercomRouteUpdate, getIntercomRouteKey } from './updates'; - -describe('Intercom updates', () => { - it('builds a minimal route update with identity and the current timestamp', () => { - const bootOptions = { - company: { id: 'organization_123', name: 'Acme' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token', - userId: 'user_123' - } as BootOptions; - - expect(buildIntercomRouteUpdate(bootOptions, 1_750_000_123_456)).toEqual({ - email: 'user@example.com', - intercomUserJwt: 'signed-token', - lastRequestAt: 1_750_000_123, - userId: 'user_123' - }); - }); - - it('includes only identity and changed user data in a data update', () => { - const previousBootOptions = { - company: { id: 'organization_123', name: 'Acme' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token-1', - userId: 'user_123' - } as BootOptions; - const bootOptions = { - company: { id: 'organization_123', name: 'Acme' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token-2', - userId: 'user_123' - } as BootOptions; - - expect(buildIntercomDataUpdate(previousBootOptions, bootOptions)).toEqual({ - email: 'user@example.com', - intercomUserJwt: 'signed-token-2', - userId: 'user_123' - }); - }); - - it('includes changed company data without resending unchanged user fields', () => { - const previousBootOptions = { - company: { id: 'organization_123', name: 'Acme' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token', - name: 'Example User', - userId: 'user_123' - } as BootOptions; - const bootOptions = { - company: { id: 'organization_123', name: 'Acme, Inc.' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token', - name: 'Example User', - userId: 'user_123' - } as BootOptions; - - expect(buildIntercomDataUpdate(previousBootOptions, bootOptions)).toEqual({ - company: { id: 'organization_123', name: 'Acme, Inc.' }, - email: 'user@example.com', - intercomUserJwt: 'signed-token', - userId: 'user_123' - }); - }); - - it('uses the normalized route ID instead of resource identifiers in the pathname', () => { - const routeId = '/(app)/project/[projectId]/event/[eventId]'; - - expect(getIntercomRouteKey(routeId, '/next/project/project-a/event/event-a')).toBe(routeId); - expect(getIntercomRouteKey(routeId, '/next/project/project-a/event/event-b')).toBe(routeId); - }); - - it('falls back to the pathname when SvelteKit has no route ID', () => { - expect(getIntercomRouteKey(null, '/next/status')).toBe('/next/status'); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts deleted file mode 100644 index 08a5deca03..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { BootOptions, UpdateOptions } from 'svelte-intercom'; - -export function buildIntercomDataUpdate(previousBootOptions: BootOptions, bootOptions: BootOptions): UpdateOptions { - const update: Record = {}; - addIntercomIdentity(update, bootOptions); - - for (const [key, value] of Object.entries(bootOptions)) { - if (key === 'email' || key === 'userId') { - continue; - } - - if (!areIntercomValuesEqual(previousBootOptions[key], value)) { - update[key] = value; - } - } - - return update as UpdateOptions; -} - -export function buildIntercomRouteUpdate(bootOptions: BootOptions, now = Date.now()): UpdateOptions { - const update: Record = { lastRequestAt: Math.floor(now / 1000) }; - addIntercomIdentity(update, bootOptions); - - // Intercom's SPA guidance requires last_request_at for URL-only updates, but - // svelte-intercom currently marks this supported field as `never` in its types. - return update as UpdateOptions; -} - -export function getIntercomRouteKey(routeId: null | string | undefined, pathname: string) { - return routeId ?? pathname; -} - -function addIntercomIdentity(update: Record, bootOptions: BootOptions) { - if (bootOptions.intercomUserJwt) { - update.intercomUserJwt = bootOptions.intercomUserJwt; - } - - if (bootOptions.email) { - update.email = bootOptions.email; - } - - if (bootOptions.userId) { - update.userId = bootOptions.userId; - } -} - -function areIntercomValuesEqual(previousValue: unknown, value: unknown) { - if (Object.is(previousValue, value)) { - return true; - } - - if (typeof previousValue !== 'object' || previousValue === null || typeof value !== 'object' || value === null) { - return false; - } - - return JSON.stringify(previousValue) === JSON.stringify(value); -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte index b76f528af6..9b3601a8bd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte @@ -72,6 +72,8 @@ if (start instanceof Date && end instanceof Date) { onRangeSelect?.(start, end); } + + e.brush.reset(); } }} props={{ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte index 269605ec44..7bdf920933 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte @@ -64,5 +64,5 @@ {:else if isNull} (Null) {:else} - {value} + {value} {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts deleted file mode 100644 index 15544e1fb4..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { render } from '@testing-library/svelte'; -import { describe, expect, it } from 'vitest'; - -import ObjectDump from './object-dump.svelte'; - -describe('ObjectDump', () => { - it('preserves line breaks in string values', () => { - const { container } = render(ObjectDump, { value: 'First line\r\nSecond line\r\nThird line' }); - - const value = container.firstElementChild; - - expect(value?.textContent).toBe('First line\r\nSecond line\r\nThird line'); - expect(value?.classList.contains('whitespace-pre-wrap')).toBe(true); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte index e930aa1cc9..8218eaf306 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte @@ -103,7 +103,7 @@ {/if} {/snippet} - +
{#snippet children(openChat)} {#if isSetupPage} diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte index bd76170456..a9261f957c 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte @@ -12,7 +12,6 @@ import { Checkbox } from '$comp/ui/checkbox'; import { Spinner } from '$comp/ui/spinner'; import { accessToken } from '$features/auth/index.svelte'; - import { clearAuthenticationSession } from '$features/auth/session.svelte'; import { getOrganizationsQuery } from '$features/organizations/api.svelte'; import { getMeQuery } from '$features/users/api.svelte'; import { useFetchClient } from '@foundatiofx/fetchclient'; @@ -225,7 +224,7 @@ } async function redirectToLogin(): Promise { - clearAuthenticationSession(); + accessToken.current = null; const returnUrl = `${page.url.pathname}${page.url.search}`; const loginUrl = `${resolve('/(auth)/login')}?redirect=${encodeURIComponent(returnUrl)}`; await goto(loginUrl, { replaceState: true }); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte index 91424cede8..8e59bac810 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte @@ -7,7 +7,6 @@ import * as Sidebar from '$comp/ui/sidebar'; import { Toaster } from '$comp/ui/sonner'; import { accessToken } from '$features/auth/index.svelte'; - import { handleUnexpectedUnauthorized } from '$features/auth/unauthorized'; import { type FetchClientContext, ProblemDetails, setAccessTokenFunc, setBaseUrl, setRequestOptions, useMiddleware } from '@foundatiofx/fetchclient'; import { error } from '@sveltejs/kit'; import { QueryClient, QueryClientProvider } from '@tanstack/svelte-query'; @@ -42,8 +41,10 @@ return; } - if (handleUnexpectedUnauthorized(status, ctx.options.expectedStatusCodes)) { - return; + if (status === 401 && !ctx.options.expectedStatusCodes?.includes(401)) { + if (accessToken.current) { + accessToken.current = ''; + } } else if (status === 404 && !ctx.options.expectedStatusCodes?.includes(404)) { throw error(404, 'Not found'); } else if ([0, 408, 503].includes(status) && !ctx.options.expectedStatusCodes?.includes(status)) { diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index f80d46d4fb..3024850883 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -286,18 +286,8 @@ ApplicationException applicationException when applicationException.Message.Cont .To("https://collector.exceptionless.io") .To("https://config.exceptionless.io") .To("https://heartbeat.exceptionless.io") - .To("https://via.intercom.io") - .To("https://api.intercom.io") .To("https://api-iam.intercom.io/") - .To("https://api-ping.intercom.io") - .To("https://*.intercom-messenger.com") - .To("wss://*.intercom-messenger.com") - .To("https://nexus-websocket-a.intercom.io") - .To("wss://nexus-websocket-a.intercom.io") - .To("https://nexus-websocket-b.intercom.io") - .To("wss://nexus-websocket-b.intercom.io") - .To("https://uploads.intercomcdn.com") - .To("https://uploads.intercomusercontent.com"); + .To("wss://nexus-websocket-a.intercom.io"); csp.OnSendingHeader = new Func(context => { From 758560ef3a545eb0ff1c89a8d0ee2db291384eab Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:35:45 -0500 Subject: [PATCH 9/9] Reapply "Merge remote-tracking branch 'origin/main' into feature/pr-2267-live-0700" This reverts commit 51465b9ee0e30e9b69444e31ffe17c14cd2e9f5b. --- .../e2e/tests/stack-effects-chaos.e2e.ts | 25 ++++++ .../src/lib/features/auth/api.svelte.ts | 11 +-- .../src/lib/features/auth/api.test.ts | 5 ++ .../src/lib/features/auth/index.svelte.ts | 15 +--- .../src/lib/features/auth/session.svelte.ts | 8 ++ .../src/lib/features/auth/state.svelte.ts | 12 +++ .../lib/features/auth/unauthorized.test.ts | 29 +++++++ .../src/lib/features/auth/unauthorized.ts | 11 +++ .../components/events-dashboard-chart.svelte | 2 - .../intercom/intercom-initializer.svelte | 66 ++++++++------- .../intercom/intercom-shell.svelte.test.ts | 84 +++++++++++++++---- .../intercom-shell.test-harness.svelte | 5 +- .../features/intercom/session.svelte.test.ts | 39 +++++++++ .../src/lib/features/intercom/session.ts | 47 +++++++++++ .../src/lib/features/intercom/updates.test.ts | 79 +++++++++++++++++ .../src/lib/features/intercom/updates.ts | 57 +++++++++++++ .../sessions-dashboard-chart.svelte | 2 - .../shared/components/object-dump.svelte | 2 +- .../components/object-dump.svelte.test.ts | 15 ++++ .../components/ui/chart/chart-tooltip.svelte | 2 +- .../ClientApp/src/routes/(app)/+layout.svelte | 3 +- .../(auth)/oauth/authorize/+page.svelte | 3 +- .../ClientApp/src/routes/+layout.svelte | 7 +- src/Exceptionless.Web/Program.cs | 12 ++- 24 files changed, 456 insertions(+), 85 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts index 6a99af1d4f..3f8a565737 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts @@ -91,6 +91,31 @@ test('stack effects stay bounded through background, paging, and navigation chao await expect(page.getByRole('heading', { name: 'Stacks' })).toBeVisible(); }); + await measureAction(diagnostics, 'stack timeline drag', async () => { + await page.locator('tbody tr:visible').first().click(); + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + const timeline = dialog.locator('[data-slot="chart"]'); + await expect(timeline).toBeVisible(); + + const bounds = await timeline.boundingBox(); + expect(bounds).not.toBeNull(); + if (bounds) { + const y = bounds.y + bounds.height / 2; + await page.mouse.move(bounds.x + bounds.width * 0.2, y); + await page.waitForTimeout(250); + await page.mouse.move(bounds.x + bounds.width * 0.3, y); + await expect(page.locator('.lc-tooltip-root')).toHaveCSS('pointer-events', 'none'); + await page.mouse.down(); + await page.mouse.move(bounds.x + bounds.width * 0.8, y, { steps: 20 }); + await page.mouse.up(); + } + + await page.waitForTimeout(500); + await page.getByRole('button', { name: 'Close' }).click(); + }); + expect(actionSample(diagnostics, 'stack timeline drag').runtimeErrors).toBe(0); + await measureAction(diagnostics, 'selected stack refresh', async () => { const rowSelection = page.getByRole('checkbox', { name: 'Select row' }).first(); await rowSelection.click(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts index 306ecf376c..ec9ed03098 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts @@ -2,13 +2,13 @@ import { env } from '$env/dynamic/public'; import { getIntercomTokenSessionKey, intercomTokenRefreshIntervalMs } from '$features/intercom/config'; import { organization } from '$features/organizations/context.svelte'; import { ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; -import { hide as hideIntercom, shutdown as shutdownIntercom } from '@intercom/messenger-js-sdk'; import { createQuery, type QueryClient } from '@tanstack/svelte-query'; import type { Login, TokenResult } from './models'; import { endSession } from './exceptionless-session'; -import { accessToken } from './index.svelte'; +import { clearAuthenticationSession } from './session.svelte'; +import { accessToken } from './state.svelte'; const queryKeys = { intercom: (accessToken: null | string) => ['Auth', 'intercom', getIntercomTokenSessionKey(accessToken)] as const @@ -103,17 +103,12 @@ export async function logout(queryClient?: QueryClient, client = useFetchClient( await queryClient?.cancelQueries(); queryClient?.clear(); - if (typeof window !== 'undefined' && 'Intercom' in window && typeof window.Intercom === 'function') { - hideIntercom(); - shutdownIntercom(); - } + clearAuthenticationSession(); organization.current = undefined; if (typeof localStorage !== 'undefined') { localStorage.removeItem('organization'); } - - accessToken.current = null; } export async function resetPassword(passwordResetToken: string, password: string) { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts index 6be26032b9..729a0a4dae 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.test.ts @@ -1,14 +1,18 @@ import { FetchClient } from '@foundatiofx/fetchclient'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +const clearAuthenticationSession = vi.hoisted(() => vi.fn()); + vi.mock('./exceptionless-session', () => ({ endSession: vi.fn() })); +vi.mock('./session.svelte', () => ({ clearAuthenticationSession })); import { logout } from './api.svelte'; describe('logout', () => { beforeEach(() => { + clearAuthenticationSession.mockReset(); // Mock localStorage for server-side tests Object.defineProperty(globalThis, 'localStorage', { configurable: true, @@ -28,5 +32,6 @@ describe('logout', () => { await logout(undefined, mockClient); expect(mockClient.get).toHaveBeenCalledWith('auth/logout', { expectedStatusCodes: [200, 401, 403] }); + expect(clearAuthenticationSession).toHaveBeenCalledOnce(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts index b0a247a3ba..0581a25cbb 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/index.svelte.ts @@ -2,11 +2,12 @@ import { goto } from '$app/navigation'; import { resolve } from '$app/paths'; import { page } from '$app/state'; import { env } from '$env/dynamic/public'; -import { CachedPersistedState } from '$features/shared/utils/cached-persisted-state.svelte'; import { useFetchClient } from '@foundatiofx/fetchclient'; import type { TokenResult } from './models'; +import { accessToken } from './state.svelte'; + // Re-export all API functions for backward compatibility export { cancelResetPassword, @@ -21,6 +22,7 @@ export { unlinkOAuthAccount } from './api.svelte'; +export { accessToken } from './state.svelte'; // Re-export validators export { validateEmailAvailability } from './validators'; @@ -44,17 +46,6 @@ export interface OAuthResponseData { export type SupportedOAuthProviders = 'facebook' | 'github' | 'google' | 'live' | 'slack'; -const authSerializer = { - deserialize: (value: null | string): null | string => { - return value === '' ? null : value; - }, - serialize: (value: null | string): string => { - return value === null ? '' : value; - } -}; - -export const accessToken = new CachedPersistedState('satellizer_token', null, { serializer: authSerializer }); - export const enableAccountCreation = env.PUBLIC_ENABLE_ACCOUNT_CREATION === 'true'; export const facebookClientId = env.PUBLIC_FACEBOOK_APPID; export const gitHubClientId = env.PUBLIC_GITHUB_APPID; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts new file mode 100644 index 0000000000..1882ade1fa --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/session.svelte.ts @@ -0,0 +1,8 @@ +import { shutdownIntercomSession } from '$features/intercom/session'; + +import { accessToken } from './state.svelte'; + +export function clearAuthenticationSession(clearedAccessToken: '' | null = null) { + shutdownIntercomSession(); + accessToken.current = clearedAccessToken; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts new file mode 100644 index 0000000000..ef9a1973dd --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/state.svelte.ts @@ -0,0 +1,12 @@ +import { CachedPersistedState } from '$features/shared/utils/cached-persisted-state.svelte'; + +const authSerializer = { + deserialize: (value: null | string): null | string => { + return value === '' ? null : value; + }, + serialize: (value: null | string): string => { + return value === null ? '' : value; + } +}; + +export const accessToken = new CachedPersistedState('satellizer_token', null, { serializer: authSerializer }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts new file mode 100644 index 0000000000..461f62c168 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.test.ts @@ -0,0 +1,29 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const clearAuthenticationSession = vi.hoisted(() => vi.fn()); +const accessToken = vi.hoisted(() => ({ current: 'token_123' as null | string })); + +vi.mock('./session.svelte', () => ({ clearAuthenticationSession })); +vi.mock('./state.svelte', () => ({ accessToken })); + +import { handleUnexpectedUnauthorized } from './unauthorized'; + +describe('handleUnexpectedUnauthorized', () => { + beforeEach(() => { + accessToken.current = 'token_123'; + clearAuthenticationSession.mockReset(); + }); + + it('clears the authenticated session after an unexpected unauthorized response', () => { + expect(handleUnexpectedUnauthorized(401)).toBe(true); + expect(clearAuthenticationSession).toHaveBeenCalledExactlyOnceWith(''); + }); + + it('ignores expected unauthorized responses and duplicate session cleanup', () => { + expect(handleUnexpectedUnauthorized(401, [401])).toBe(false); + + accessToken.current = null; + expect(handleUnexpectedUnauthorized(401)).toBe(false); + expect(clearAuthenticationSession).not.toHaveBeenCalled(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts new file mode 100644 index 0000000000..7174a30448 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/unauthorized.ts @@ -0,0 +1,11 @@ +import { clearAuthenticationSession } from './session.svelte'; +import { accessToken } from './state.svelte'; + +export function handleUnexpectedUnauthorized(status: number, expectedStatusCodes?: number[]) { + if (status !== 401 || expectedStatusCodes?.includes(401) || !accessToken.current) { + return false; + } + + clearAuthenticationSession(''); + return true; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte index 7be0e2b734..869049ccb7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-dashboard-chart.svelte @@ -72,8 +72,6 @@ if (start instanceof Date && end instanceof Date) { onRangeSelect?.(start, end); } - - e.brush.reset(); } }} props={{ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte index 036b98951e..5803befbd5 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-initializer.svelte @@ -8,66 +8,68 @@ import type { Snippet } from 'svelte'; import type { BootOptions } from 'svelte-intercom'; - import { accessToken } from '$features/auth/index.svelte'; - import { DocumentVisibility } from '$shared/document-visibility.svelte'; - import { useInterval } from 'runed'; import { untrack } from 'svelte'; import { setContext } from 'svelte'; import { useIntercom } from 'svelte-intercom'; import { INTERCOM_CONTEXT_KEY } from './keys'; + import { buildIntercomDataUpdate, buildIntercomRouteUpdate } from './updates'; interface Props { bootOptions?: BootOptions; children: Snippet; routeKey?: string; + /** @deprecated Intercom updates are event-driven; this value is retained as a no-op for compatibility. */ updateIntervalMs?: number; } - let { bootOptions = undefined, children, routeKey = undefined, updateIntervalMs = 90_000 }: Props = $props(); + let { bootOptions = undefined, children, routeKey = undefined, updateIntervalMs = undefined }: Props = $props(); const intercom = useIntercom(); - const visibility = new DocumentVisibility(); + let hasBooted = false; + let previousBootOptions: BootOptions | undefined; + let previousRouteKey: string | undefined; setContext(INTERCOM_CONTEXT_KEY, intercom); - const interval = useInterval(() => updateIntervalMs, { - callback: () => { - if (bootOptions && visibility.visible) { - intercom.update(bootOptions); - } - }, - immediate: false + // Retain the deprecated prop reactively without restoring periodic updates. + $effect(() => { + void updateIntervalMs; }); - const shouldUpdate = $derived(bootOptions && visibility.visible); - - // Sync identity/company data and manage interval when boot options or visibility changes. + // The provider boots with the initial options. Only update after boot when the route or + // identity/company data changes; eager or periodic updates create duplicate impressions. $effect(() => { - if (!bootOptions) { - interval.pause(); + const options = bootOptions; + const currentRouteKey = routeKey; + if (!options) { + hasBooted = false; + previousBootOptions = undefined; + previousRouteKey = undefined; return; } - if (visibility.visible) { - interval.resume(); - } else { - interval.pause(); + if (!hasBooted) { + hasBooted = true; + previousBootOptions = options; + previousRouteKey = currentRouteKey; + return; } - }); - // Sync on route transitions and visibility changes. - $effect(() => { - void routeKey; - if (shouldUpdate) { - untrack(() => intercom.update(bootOptions!)); + if (typeof window.Intercom !== 'function') { + return; } - }); - // Shutdown when the user logs out. - $effect(() => { - if (!accessToken.current) { - untrack(() => intercom.shutdown()); + const priorBootOptions = previousBootOptions!; + const bootOptionsChanged = options !== priorBootOptions; + const routeChanged = currentRouteKey !== previousRouteKey; + previousBootOptions = options; + previousRouteKey = currentRouteKey; + + if (bootOptionsChanged) { + untrack(() => intercom.update(buildIntercomDataUpdate(priorBootOptions, options))); + } else if (routeChanged) { + untrack(() => intercom.update(buildIntercomRouteUpdate(options))); } }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts index dbeb4f03ea..a09ab0e06d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.svelte.test.ts @@ -43,6 +43,7 @@ describe('IntercomShell', () => { intercomShowMessages.mockReset(); intercomUpdate.mockReset(); vi.restoreAllMocks(); + window.Intercom = vi.fn(); }); it('keeps children mounted when Intercom becomes bootable', async () => { @@ -76,37 +77,84 @@ describe('IntercomShell', () => { expect(intercomShowMessages).toHaveBeenCalledTimes(1); }); - it('remains stable across repeated tab visibility changes', async () => { + it('updates after boot only when the route or boot options change', async () => { // Arrange - let hidden = false; - const addEventListener = vi.spyOn(document, 'addEventListener'); - vi.spyOn(document, 'hidden', 'get').mockImplementation(() => hidden); + const bootOptions = { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions; const { rerender } = render(IntercomShellTestHarness, { props: { appId: 'app_123', - bootOptions: { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions + bootOptions, + routeKey: '/event/all' } }); await tick(); + // Assert initial boot options are not immediately sent again. + expect(intercomUpdate).not.toHaveBeenCalled(); + + // Act + await rerender({ appId: 'app_123', bootOptions, routeKey: '/stack/all' }); + + // Assert + expect(intercomUpdate).toHaveBeenCalledOnce(); + expect(intercomUpdate).toHaveBeenLastCalledWith({ + intercom_user_jwt: 'token_0', + last_request_at: expect.any(Number), + user_id: 'user_123' + }); + // Act - for (let index = 0; index < 100; index++) { - hidden = true; - document.dispatchEvent(new Event('visibilitychange')); - await tick(); + await rerender({ + appId: 'app_123', + bootOptions: { intercomUserJwt: 'token_1', userId: 'user_123' } as BootOptions, + routeKey: '/stack/all' + }); + + // Assert + expect(intercomUpdate).toHaveBeenCalledTimes(2); + expect(intercomUpdate).toHaveBeenLastCalledWith({ + intercom_user_jwt: 'token_1', + user_id: 'user_123' + }); + }); - await rerender({ + it('does not update when navigation stays within the same normalized route', async () => { + const bootOptions = { email: 'user@example.com', userId: 'user_123' } as BootOptions; + const routeKey = '/(app)/project/[projectId]/event/[eventId]'; + const { rerender } = render(IntercomShellTestHarness, { + props: { appId: 'app_123', bootOptions, routeKey } + }); + await tick(); + + await rerender({ appId: 'app_123', bootOptions, routeKey }); + + expect(intercomUpdate).not.toHaveBeenCalled(); + }); + + it('does not update before the client SDK initializes', async () => { + // Arrange + window.Intercom = undefined; + const bootOptions = { intercomUserJwt: 'token_0', userId: 'user_123' } as BootOptions; + const { rerender } = render(IntercomShellTestHarness, { + props: { appId: 'app_123', - bootOptions: { intercomUserJwt: `token_${index + 1}`, userId: 'user_123' } as BootOptions - }); + bootOptions, + routeKey: '/event/all' + } + }); + await tick(); + + // Act + await rerender({ appId: 'app_123', bootOptions, routeKey: '/stack/all' }); - hidden = false; - document.dispatchEvent(new Event('visibilitychange')); - await tick(); - } + // Assert + expect(intercomUpdate).not.toHaveBeenCalled(); + + // Act + window.Intercom = vi.fn(); + await rerender({ appId: 'app_123', bootOptions, routeKey: '/event/errors' }); // Assert - expect(intercomUpdate).toHaveBeenCalled(); - expect(addEventListener.mock.calls.filter(([eventName]) => eventName === 'visibilitychange')).toHaveLength(1); + expect(intercomUpdate).toHaveBeenCalledOnce(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte index 8ff9d77876..5e1655bc38 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/intercom-shell.test-harness.svelte @@ -8,12 +8,13 @@ appId?: string; bootOptions?: BootOptions; onMountProbe?: () => void; + routeKey?: string; } - let { appId = undefined, bootOptions = undefined, onMountProbe = () => {} }: Props = $props(); + let { appId = undefined, bootOptions = undefined, onMountProbe = () => {}, routeKey = undefined }: Props = $props(); - + {#snippet children(openChat)} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts new file mode 100644 index 0000000000..113ad71707 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.svelte.test.ts @@ -0,0 +1,39 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const hide = vi.hoisted(() => vi.fn()); +const shutdown = vi.hoisted(() => vi.fn()); + +vi.mock('@intercom/messenger-js-sdk', () => ({ hide, shutdown })); + +import { shutdownIntercomSession } from './session'; + +describe('shutdownIntercomSession', () => { + beforeEach(() => { + hide.mockReset(); + shutdown.mockReset(); + window.Intercom = vi.fn(); + document.cookie = 'intercom-id-app=visitor; Path=/'; + document.cookie = 'intercom-session-app=session; Path=/'; + document.cookie = 'unrelated-cookie=keep; Path=/'; + }); + + it('shuts down the SDK and clears all Intercom cookies', () => { + shutdownIntercomSession(); + + expect(hide).toHaveBeenCalledOnce(); + expect(shutdown).toHaveBeenCalledOnce(); + expect(document.cookie).not.toContain('intercom-id-app'); + expect(document.cookie).not.toContain('intercom-session-app'); + expect(document.cookie).toContain('unrelated-cookie=keep'); + }); + + it('still clears cookies when tracking prevention blocks the SDK', () => { + window.Intercom = undefined; + + expect(() => shutdownIntercomSession()).not.toThrow(); + expect(hide).not.toHaveBeenCalled(); + expect(shutdown).not.toHaveBeenCalled(); + expect(document.cookie).not.toContain('intercom-id-app'); + expect(document.cookie).not.toContain('intercom-session-app'); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts new file mode 100644 index 0000000000..2c5f7082ad --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/session.ts @@ -0,0 +1,47 @@ +import { hide, shutdown } from '@intercom/messenger-js-sdk'; + +export function clearIntercomCookies() { + if (typeof document === 'undefined') { + return; + } + + const cookieNames = new Set( + document.cookie + .split(';') + .map((cookie) => cookie.trim().split('=', 1)[0] ?? '') + .filter((name) => name.startsWith('intercom-')) + ); + + const hostname = typeof window === 'undefined' ? '' : window.location.hostname; + const domainCandidates = getCookieDomainCandidates(hostname); + + for (const name of cookieNames) { + expireCookie(name); + for (const domain of domainCandidates) { + expireCookie(name, domain); + } + } +} + +export function shutdownIntercomSession() { + if (typeof window !== 'undefined' && typeof window.Intercom === 'function') { + hide(); + shutdown(); + } + + clearIntercomCookies(); +} + +function expireCookie(name: string, domain?: string) { + const domainAttribute = domain ? `; Domain=${domain}` : ''; + document.cookie = `${name}=; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/${domainAttribute}; SameSite=Lax`; +} + +function getCookieDomainCandidates(hostname: string) { + if (!hostname.includes('.') || /^\d{1,3}(\.\d{1,3}){3}$/.test(hostname)) { + return []; + } + + const parts = hostname.split('.'); + return parts.slice(0, -1).map((_, index) => parts.slice(index).join('.')); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts new file mode 100644 index 0000000000..646684f360 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.test.ts @@ -0,0 +1,79 @@ +import type { BootOptions } from 'svelte-intercom'; + +import { describe, expect, it } from 'vitest'; + +import { buildIntercomDataUpdate, buildIntercomRouteUpdate, getIntercomRouteKey } from './updates'; + +describe('Intercom updates', () => { + it('builds a minimal route update with identity and the current timestamp', () => { + const bootOptions = { + company: { id: 'organization_123', name: 'Acme' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token', + userId: 'user_123' + } as BootOptions; + + expect(buildIntercomRouteUpdate(bootOptions, 1_750_000_123_456)).toEqual({ + email: 'user@example.com', + intercomUserJwt: 'signed-token', + lastRequestAt: 1_750_000_123, + userId: 'user_123' + }); + }); + + it('includes only identity and changed user data in a data update', () => { + const previousBootOptions = { + company: { id: 'organization_123', name: 'Acme' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token-1', + userId: 'user_123' + } as BootOptions; + const bootOptions = { + company: { id: 'organization_123', name: 'Acme' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token-2', + userId: 'user_123' + } as BootOptions; + + expect(buildIntercomDataUpdate(previousBootOptions, bootOptions)).toEqual({ + email: 'user@example.com', + intercomUserJwt: 'signed-token-2', + userId: 'user_123' + }); + }); + + it('includes changed company data without resending unchanged user fields', () => { + const previousBootOptions = { + company: { id: 'organization_123', name: 'Acme' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token', + name: 'Example User', + userId: 'user_123' + } as BootOptions; + const bootOptions = { + company: { id: 'organization_123', name: 'Acme, Inc.' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token', + name: 'Example User', + userId: 'user_123' + } as BootOptions; + + expect(buildIntercomDataUpdate(previousBootOptions, bootOptions)).toEqual({ + company: { id: 'organization_123', name: 'Acme, Inc.' }, + email: 'user@example.com', + intercomUserJwt: 'signed-token', + userId: 'user_123' + }); + }); + + it('uses the normalized route ID instead of resource identifiers in the pathname', () => { + const routeId = '/(app)/project/[projectId]/event/[eventId]'; + + expect(getIntercomRouteKey(routeId, '/next/project/project-a/event/event-a')).toBe(routeId); + expect(getIntercomRouteKey(routeId, '/next/project/project-a/event/event-b')).toBe(routeId); + }); + + it('falls back to the pathname when SvelteKit has no route ID', () => { + expect(getIntercomRouteKey(null, '/next/status')).toBe('/next/status'); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts new file mode 100644 index 0000000000..08a5deca03 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/intercom/updates.ts @@ -0,0 +1,57 @@ +import type { BootOptions, UpdateOptions } from 'svelte-intercom'; + +export function buildIntercomDataUpdate(previousBootOptions: BootOptions, bootOptions: BootOptions): UpdateOptions { + const update: Record = {}; + addIntercomIdentity(update, bootOptions); + + for (const [key, value] of Object.entries(bootOptions)) { + if (key === 'email' || key === 'userId') { + continue; + } + + if (!areIntercomValuesEqual(previousBootOptions[key], value)) { + update[key] = value; + } + } + + return update as UpdateOptions; +} + +export function buildIntercomRouteUpdate(bootOptions: BootOptions, now = Date.now()): UpdateOptions { + const update: Record = { lastRequestAt: Math.floor(now / 1000) }; + addIntercomIdentity(update, bootOptions); + + // Intercom's SPA guidance requires last_request_at for URL-only updates, but + // svelte-intercom currently marks this supported field as `never` in its types. + return update as UpdateOptions; +} + +export function getIntercomRouteKey(routeId: null | string | undefined, pathname: string) { + return routeId ?? pathname; +} + +function addIntercomIdentity(update: Record, bootOptions: BootOptions) { + if (bootOptions.intercomUserJwt) { + update.intercomUserJwt = bootOptions.intercomUserJwt; + } + + if (bootOptions.email) { + update.email = bootOptions.email; + } + + if (bootOptions.userId) { + update.userId = bootOptions.userId; + } +} + +function areIntercomValuesEqual(previousValue: unknown, value: unknown) { + if (Object.is(previousValue, value)) { + return true; + } + + if (typeof previousValue !== 'object' || previousValue === null || typeof value !== 'object' || value === null) { + return false; + } + + return JSON.stringify(previousValue) === JSON.stringify(value); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte index 9b3601a8bd..b76f528af6 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/sessions-dashboard-chart.svelte @@ -72,8 +72,6 @@ if (start instanceof Date && end instanceof Date) { onRangeSelect?.(start, end); } - - e.brush.reset(); } }} props={{ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte index 7bdf920933..269605ec44 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte @@ -64,5 +64,5 @@ {:else if isNull} (Null) {:else} - {value} + {value} {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts new file mode 100644 index 0000000000..15544e1fb4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/object-dump.svelte.test.ts @@ -0,0 +1,15 @@ +import { render } from '@testing-library/svelte'; +import { describe, expect, it } from 'vitest'; + +import ObjectDump from './object-dump.svelte'; + +describe('ObjectDump', () => { + it('preserves line breaks in string values', () => { + const { container } = render(ObjectDump, { value: 'First line\r\nSecond line\r\nThird line' }); + + const value = container.firstElementChild; + + expect(value?.textContent).toBe('First line\r\nSecond line\r\nThird line'); + expect(value?.classList.contains('whitespace-pre-wrap')).toBe(true); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte index 8218eaf306..e930aa1cc9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/chart/chart-tooltip.svelte @@ -103,7 +103,7 @@ {/if} {/snippet} - +
{#snippet children(openChat)} {#if isSetupPage} diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte index a9261f957c..bd76170456 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(auth)/oauth/authorize/+page.svelte @@ -12,6 +12,7 @@ import { Checkbox } from '$comp/ui/checkbox'; import { Spinner } from '$comp/ui/spinner'; import { accessToken } from '$features/auth/index.svelte'; + import { clearAuthenticationSession } from '$features/auth/session.svelte'; import { getOrganizationsQuery } from '$features/organizations/api.svelte'; import { getMeQuery } from '$features/users/api.svelte'; import { useFetchClient } from '@foundatiofx/fetchclient'; @@ -224,7 +225,7 @@ } async function redirectToLogin(): Promise { - accessToken.current = null; + clearAuthenticationSession(); const returnUrl = `${page.url.pathname}${page.url.search}`; const loginUrl = `${resolve('/(auth)/login')}?redirect=${encodeURIComponent(returnUrl)}`; await goto(loginUrl, { replaceState: true }); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte index 8e59bac810..91424cede8 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/+layout.svelte @@ -7,6 +7,7 @@ import * as Sidebar from '$comp/ui/sidebar'; import { Toaster } from '$comp/ui/sonner'; import { accessToken } from '$features/auth/index.svelte'; + import { handleUnexpectedUnauthorized } from '$features/auth/unauthorized'; import { type FetchClientContext, ProblemDetails, setAccessTokenFunc, setBaseUrl, setRequestOptions, useMiddleware } from '@foundatiofx/fetchclient'; import { error } from '@sveltejs/kit'; import { QueryClient, QueryClientProvider } from '@tanstack/svelte-query'; @@ -41,10 +42,8 @@ return; } - if (status === 401 && !ctx.options.expectedStatusCodes?.includes(401)) { - if (accessToken.current) { - accessToken.current = ''; - } + if (handleUnexpectedUnauthorized(status, ctx.options.expectedStatusCodes)) { + return; } else if (status === 404 && !ctx.options.expectedStatusCodes?.includes(404)) { throw error(404, 'Not found'); } else if ([0, 408, 503].includes(status) && !ctx.options.expectedStatusCodes?.includes(status)) { diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 3024850883..f80d46d4fb 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -286,8 +286,18 @@ ApplicationException applicationException when applicationException.Message.Cont .To("https://collector.exceptionless.io") .To("https://config.exceptionless.io") .To("https://heartbeat.exceptionless.io") + .To("https://via.intercom.io") + .To("https://api.intercom.io") .To("https://api-iam.intercom.io/") - .To("wss://nexus-websocket-a.intercom.io"); + .To("https://api-ping.intercom.io") + .To("https://*.intercom-messenger.com") + .To("wss://*.intercom-messenger.com") + .To("https://nexus-websocket-a.intercom.io") + .To("wss://nexus-websocket-a.intercom.io") + .To("https://nexus-websocket-b.intercom.io") + .To("wss://nexus-websocket-b.intercom.io") + .To("https://uploads.intercomcdn.com") + .To("https://uploads.intercomusercontent.com"); csp.OnSendingHeader = new Func(context => {