diff --git a/docs/_config.ts b/docs/_config.ts index 20f2af36dc..9e086a1604 100644 --- a/docs/_config.ts +++ b/docs/_config.ts @@ -36,6 +36,13 @@ site.add("public/assets/img/dashboard-2-1024x594.png", "assets/img/dashboard-2-1 site.add("public/assets/img/toexceptionless.png", "assets/img/toexceptionless.png") site.add("public/assets/img/logs-2.jpg", "assets/img/logs-2.jpg") site.add("public/assets/img/slider-github.jpg", "assets/img/slider-github.jpg") +site.add("public/assets/img/docs/configuration-precedence.svg", "assets/img/docs/configuration-precedence.svg") +site.add( + "public/assets/img/docs/infrastructure-role-selection.svg", + "assets/img/docs/infrastructure-role-selection.svg", +) +site.add("public/assets/img/docs/redis-connection-ownership.svg", "assets/img/docs/redis-connection-ownership.svg") +site.add("public/assets/img/docs/helm-version-rollout.svg", "assets/img/docs/helm-version-rollout.svg") site.hooks.markdownIt((markdownIt: any) => { const defaultHeadingOpen = markdownIt.renderer.rules.heading_open ?? diff --git a/docs/docs/self-hosting/configuration.md b/docs/docs/self-hosting/configuration.md new file mode 100644 index 0000000000..20154fb1c5 --- /dev/null +++ b/docs/docs/self-hosting/configuration.md @@ -0,0 +1,129 @@ +--- +title: "Infrastructure Configuration" +--- + +# Infrastructure Configuration + +Exceptionless can infer its cache, message bus, queue, and file storage from technology-named connection strings. Define a technology once and each compatible role will use it automatically. + +Configuration sources use the normal application precedence: command-line arguments override `EX_` environment variables, which override ordinary environment variables (including Aspire-injected variables), which override environment-specific and base YAML files. If both `EX_ConnectionStrings__Redis` and `ConnectionStrings__Redis` are present, the `EX_` value wins. + +## What changed + +This model adds automatic provider selection without changing the public option types. New configurations declare technology connection strings only. Existing Helm, Docker Compose, and self-hosted role selectors remain supported as a compatibility layer. The runtime also keeps Redis connections isolated by their effective connection string, so a legacy role-specific endpoint cannot leak into another role. + +![Configuration sources override one another before explicit role selectors or automatic priorities are evaluated.](/assets/img/docs/configuration-precedence.svg) + +## Automatic role selection + +| Role | Automatic priority | +| --- | --- | +| Cache | `Redis`, then local memory | +| MessageBus | `RabbitMQ`, then `Redis`, then local memory | +| Queue | `AzureQueues`, then `SQS`, then `Redis`, then local memory | +| Storage | `AzureStorage`, then `S3`, then `Aliyun`, then `Folder`, then local memory | + +Redis is never selected for file storage. For production file storage, configure durable Azure Blob, S3, Aliyun, or folder storage. + +The first configured technology in each role's row wins. Technology connection strings are atomic: extend or replace `ConnectionStrings:Redis` itself rather than defining a second Cache or Queue connection string. Existing `ConnectionStrings:Cache`, `MessageBus`, `Queue`, and `Storage` values still win when present so deployed Helm and Docker configurations remain safe, but they are no longer the recommended configuration model. + +![Automatic infrastructure role selection priorities, with Redis intentionally excluded from Storage.](/assets/img/docs/infrastructure-role-selection.svg) + +An existing explicit role selector short-circuits this graph. A blank legacy role value is treated as absent. + +## Examples + +Environment variables use double underscores where YAML or .NET configuration uses colons. + +### Redis only + +This uses Redis for Cache, MessageBus, and Queue. Storage remains local. + +```yaml +EX_ConnectionStrings__Redis: redis:6379,abortConnect=false +``` + +### Redis with RabbitMQ + +RabbitMQ automatically becomes MessageBus while Redis continues to supply Cache and Queue. + +```yaml +EX_ConnectionStrings__Redis: redis:6379,abortConnect=false +EX_ConnectionStrings__RabbitMQ: amqps://user:password@rabbitmq:5671/%2F +``` + +To use Redis for MessageBus instead, omit the `RabbitMQ` technology connection string. Existing deployments may retain `EX_ConnectionStrings__MessageBus=provider=redis` as a compatibility override. + +Legacy and inline RabbitMQ forms remain supported: + +```yaml +EX_ConnectionStrings__MessageBus: 'provider=rabbitmq;server="amqps://user:password@rabbitmq:5671/%2F"' +# Or: 'provider=rabbitmq;amqps://user:password@rabbitmq:5671/%2F' +``` + +Percent-encode reserved characters in RabbitMQ usernames, passwords, and virtual hosts. + +### Queue and storage technologies + +```yaml +# Azure Queue Storage is inferred for Queue. +EX_ConnectionStrings__AzureQueues: DefaultEndpointsProtocol=https;AccountName=example;AccountKey=secret + +# Azure Blob Storage is inferred for Storage. +EX_ConnectionStrings__AzureStorage: DefaultEndpointsProtocol=https;AccountName=example;AccountKey=secret + +# Folder is a named local storage technology. +EX_ConnectionStrings__Folder: path=/app/storage +``` + +SQS and S3 use `EX_ConnectionStrings__SQS` and `EX_ConnectionStrings__S3`. Aliyun storage uses `EX_ConnectionStrings__Aliyun`. Do not configure a higher-priority technology for the same role unless that priority is intentional. + +Redis and RabbitMQ native connection strings are opaque. Options belong in the complete technology connection string and are never generically concatenated with another role string. Existing `provider=...` selectors and full inline values remain supported for compatibility. New providers must add an allowlisted technology alias, compatible roles, parsing rules, and a fixed priority. + +The Redis registration follows the same layering boundary: + +![Redis connections are deduplicated only when roles resolve to the same exact connection string; WebSocket mapping always uses the Cache connection.](/assets/img/docs/redis-connection-ownership.svg) + +Equal effective strings are deduplicated; different legacy role endpoints remain isolated. Redis telemetry is enabled whenever Redis supplies any role. + +### Legacy role controls + +```yaml +EX_ConnectionStrings__Cache: local +EX_ConnectionStrings__MessageBus: local +EX_ConnectionStrings__Queue: local +EX_ConnectionStrings__Storage: local +``` + +These role keys are retained for upgrades and exceptional compatibility needs; new provider-free configurations normally omit them. `local` storage is in memory. Use the named `Folder` technology when local storage must survive restarts. + +## Helm, Docker Compose, and Aspire + +Existing Helm values and Docker Compose configurations do not need to change. Their explicit `Cache`, `MessageBus`, `Queue`, and `Storage` selectors have the highest role-selection precedence and remain supported. Helm's folder storage setting and persistent-volume behavior are unchanged. + +The current Helm chart deliberately operates in legacy-selector mode: it renders explicit values for all four roles. Consequently, adding `EX_ConnectionStrings__RabbitMQ` by itself does **not** switch MessageBus to RabbitMQ because the chart's explicit `MessageBus` value wins. Configure RabbitMQ through the existing Helm value instead: + +```yaml +messagebus: + connectionString: 'provider=rabbitmq;server="amqps://user:password@rabbitmq:5671/%2F"' +``` + +Do not set any distributed Helm role to `local` when multiple app or job replicas may run. In-memory caches, message buses, queues, and storage are process-local and cannot coordinate replicas. + +Aspire injects connection strings as `ConnectionStrings__{resource-name}` environment variables. The existing `Redis`, `AzureStorage`, and `AzureQueues` resource names therefore become `ConnectionStrings__Redis`, `ConnectionStrings__AzureStorage`, and `ConnectionStrings__AzureQueues`, which match the automatic selection aliases. Aspire does not need role selectors. An `EX_ConnectionStrings__{name}` value still wins when both forms are present. + +Elasticsearch, email, OAuth, LDAP, and other fixed-service connection strings are not part of infrastructure role selection. + +## Rolling out the change + +A rolling **version-only** Helm upgrade is compatible with mixed old and new Exceptionless instances when every rendered role selector and every effective connection string remains exactly unchanged. Keep the existing `Cache`, `MessageBus`, `Queue`, and `Storage` values in place while upgrading the images. All overlapping instances then use the same providers and endpoints. + +This compatibility statement covers the Exceptionless app and job workloads. A production installation also needs durable, highly available infrastructure; the chart's bundled single-replica Redis and Elasticsearch resources are convenience dependencies, not a zero-downtime production topology. + +Removing a selector is safe during normal operation only when the role resolves to the same provider **and the same effective connection string** before and after removal. Compare the resolved result, not merely the technology name. For example, removing `MessageBus=provider=redis` is safe only if automatic selection still chooses Redis with the identical Redis connection string. + +Changing a role's provider or endpoint is an infrastructure migration, not a zero-downtime configuration cleanup. During a rolling change, old and new replicas would otherwise be split across message buses, queue backlogs, storage locations, distributed caches, locks, and WebSocket mappings. Use a provider-specific bridge or dual-read/write migration where the technology supports it, or quiesce producers, drain outstanding work, switch every replica together during a maintenance window, and verify the new backend before resuming traffic. + +Do not combine a binary upgrade with a selector, provider, or endpoint migration. If a configuration migration must be rolled back, restore the old selector and endpoint first and wait for all replicas to converge before rolling back the application version. + +![A safe Helm image rollout keeps selectors and effective endpoints unchanged; provider or endpoint changes follow a separate migration path.](/assets/img/docs/helm-version-rollout.svg) diff --git a/docs/docs/self-hosting/docker.md b/docs/docs/self-hosting/docker.md index 32f839afc2..2a749293f0 100644 --- a/docs/docs/self-hosting/docker.md +++ b/docs/docs/self-hosting/docker.md @@ -6,6 +6,8 @@ title: "Docker" If you would like to test Exceptionless locally, please follow this section. +See [Infrastructure Configuration](/docs/self-hosting/configuration) for technology-named Redis, RabbitMQ, queue, and storage connection strings. Existing Docker Compose selectors remain supported as compatibility settings. + ## Requirements * [Docker](https://www.docker.com) diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index 829ae550b0..050ff2d035 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -10,6 +10,7 @@ You can also use Kubernetes while self-hosting Exceptionless. We'll cover both t * [Docker](/docs/self-hosting/docker) * [Kubernetes](/docs/self-hosting/kubernetes) +* [Infrastructure configuration](/docs/self-hosting/configuration) * [Upgrading](/docs/self-hosting/upgrading-self-hosted-instance) --- diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index 667ab9a4de..ee2686046f 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -11,7 +11,7 @@ Please follow this section to set up Exceptionless in a Kubernetes environment. ## Instructions -Please note that we recommend you use Kubernetes for running in production. +Please note that we recommend you use Kubernetes for running in production. Configure durable, highly available Redis and Elasticsearch services for a production installation. The chart's bundled single-replica dependencies are intended for evaluation and do not provide a zero-downtime or durable production topology. 1. Follow the steps [here](https://github.com/exceptionless/Exceptionless/blob/master/k8s/ex-setup.ps1) for how to create it in AKS 2. View the configuration settings below for more information on configuring Exceptionless. @@ -29,22 +29,29 @@ _Please note that if you are specifying configuration via `docker-compose`, then ## ConnectionStrings +See [Infrastructure Configuration](/docs/self-hosting/configuration) for the technology priority table, RabbitMQ examples, compatibility controls, Aspire naming, and rollout guidance. Existing Helm values remain supported without changes. + +The current chart renders explicit legacy selectors for Cache, MessageBus, Queue, and Storage. Those selectors intentionally override automatic technology selection. Adding `EX_ConnectionStrings__RabbitMQ` under `config` alone therefore does not switch MessageBus. Use the existing Helm value: + ```yaml -# connection string used for any provider specifying Redis. +messagebus: + connectionString: 'provider=rabbitmq;server="amqps://user:password@rabbitmq:5671/%2F"' +``` + +Keep every selector and effective endpoint unchanged during a rolling image upgrade. Changing a selector, provider, or endpoint is a separate infrastructure migration and requires a bridge or dual-read/write process, or a quiesce-and-drain maintenance window. Do not use `local` for a distributed role when multiple replicas may run; each replica would receive isolated in-memory state. + +```yaml +# Redis automatically supplies Cache, MessageBus, and Queue when a higher-priority +# technology for a role is not configured. EX_ConnectionStrings__Redis: localhost:6379,abortConnect=false -EX_ConnectionStrings__Cache: provider=redis; EX_ConnectionStrings__Elasticsearch: server=http://10.0.0.4:9200; EX_ConnectionStrings__Email: smtps://user%40domain.com:password@smtp.domain.com:465 -EX_ConnectionStrings__MessageBus: provider=redis; -EX_ConnectionStrings__Metrics: provider=statsd;server=localhost -EX_ConnectionStrings__Queue: provider=redis; -EX_ConnectionStrings__Storage: provider=azurestorage; +EX_ConnectionStrings__AzureQueues: DefaultEndpointsProtocol=https;AccountName=example;AccountKey=secret +EX_ConnectionStrings__AzureStorage: DefaultEndpointsProtocol=https;AccountName=example;AccountKey=secret ``` -You can append values to any connection string using a `;`. For example, you can control many shards and replicas each Elasticsearch index should be created with by appending to the `EX_ConnectionStrings__Elasticsearch` connection string. For a Elasticsearch cluster (3 nodes, two masters), you would append `shards=3;replicas=1`. - -The `provider` value determines what implementations to use for the various abstractions. We've made it easier to reuse a single connection string by automatically looking up a connection string by the provider name and adding any key value pairs to the current connection string (as shown above with redis). +Structured connection strings support provider-specific key-value options. For example, you can control how many shards and replicas each Elasticsearch index should be created with by appending `shards=3;replicas=1` to `EX_ConnectionStrings__Elasticsearch`. Redis and RabbitMQ use their native complete connection-string formats. ## General Configuration diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index dd0974989f..f974df6d9e 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -6,6 +6,14 @@ title: "Upgrading" **Please ensure that you have created backups before upgrading!** +## Layered infrastructure connection strings + +Existing Helm values, Docker Compose settings, and explicit `provider=...` role connection strings require no changes. They continue to override automatic provider selection. New installations may use the simpler technology-named connection strings described in [Infrastructure Configuration](/docs/self-hosting/configuration). + +For a rolling Helm image upgrade, leave every rendered role selector and effective connection string exactly unchanged while old and new replicas overlap. The current chart remains in legacy-selector mode, so a technology-named RabbitMQ connection string alone does not override its explicit MessageBus value. Configure RabbitMQ through `messagebus.connectionString` as documented in [Infrastructure Configuration](/docs/self-hosting/configuration#helm-docker-compose-and-aspire). + +Treat selector removal and provider or endpoint changes as separate infrastructure migrations. Selector removal is safe only when the resolved provider and exact effective connection string are identical before and after the change. A real backend change requires a provider-specific bridge or dual-read/write process, or a quiesce-and-drain maintenance window; it is not a zero-downtime rolling configuration change. + **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** ## Upgrading from v7.1 to v8 diff --git a/docs/public/assets/img/docs/configuration-precedence.svg b/docs/public/assets/img/docs/configuration-precedence.svg new file mode 100644 index 0000000000..08e046e5ee --- /dev/null +++ b/docs/public/assets/img/docs/configuration-precedence.svg @@ -0,0 +1,73 @@ + + Exceptionless configuration precedence and role resolution + Base YAML is overridden by environment YAML, ordinary and Aspire environment variables, EX-prefixed environment variables, and command-line arguments. After the effective configuration is built, a nonblank explicit role selector wins; otherwise the fixed technology priority is evaluated and falls back to local memory. + + + + + + + + Configuration is layered first, then each role is resolved + Sources farther to the right override earlier sources after keys are normalized. + + + Base YAML + lowest precedence + + + Environment YAML + local or environment + + + Ordinary variables + including Aspire + + + EX_ variables + wins over Aspire + + + Command line + highest precedence + + + + + + + + Effective configuration + blank role values count as absent + + + + Nonblank explicit + role value? + + + + Yes: validate and use it + explicit selector or local wins + + yes + + + No: fixed provider priority + first configured compatible technology + + no + + + None configured: local memory + + diff --git a/docs/public/assets/img/docs/helm-version-rollout.svg b/docs/public/assets/img/docs/helm-version-rollout.svg new file mode 100644 index 0000000000..787219003b --- /dev/null +++ b/docs/public/assets/img/docs/helm-version-rollout.svg @@ -0,0 +1,82 @@ + + Safe Helm version rollout versus infrastructure migration + The safe rolling path keeps the current chart's explicit role selectors and exact effective connection strings unchanged, upgrades only the application images, verifies mixed replicas use the same backends, and completes the rollout. Changing a selector, provider, or endpoint is a separate migration requiring a bridge or dual-read-write process, or quiescing and draining work before switching all replicas. + + + + + + + + + + + Keep a version rollout separate from a backend migration + + Current chart: explicit legacy role selectors + + Rolling image upgrade + + Preflight + same role selectors and + exact effective endpoints + + + Upgrade images only + old and new replicas + overlap normally + + + Verify overlap + every replica uses the + same providers and endpoints + + + Complete rollout + selectors remain in place + until a separate decision + + + + + + + Separate infrastructure migration — not a rolling configuration cleanup + + + Selector, provider, or + endpoint must change + plan per role and backend + + + Bridge or dual read/write + when the provider supports it + otherwise quiesce and drain + + + Switch together + move every replica to + the new backend + + + Verify + then resume + traffic + + + + + + + Configuration rollback order + Restore the old selector and endpoint → wait for every replica to converge → then roll back the binary. + diff --git a/docs/public/assets/img/docs/infrastructure-role-selection.svg b/docs/public/assets/img/docs/infrastructure-role-selection.svg new file mode 100644 index 0000000000..41a6336ad0 --- /dev/null +++ b/docs/public/assets/img/docs/infrastructure-role-selection.svg @@ -0,0 +1,79 @@ + + Automatic infrastructure role selection priorities + Cache selects Redis then local. MessageBus selects RabbitMQ, Redis, then local. Queue selects AzureQueues, SQS, Redis, then local. Storage selects AzureStorage, S3, Aliyun, Folder, then local. Redis is never selected for Storage. + + + + + + + + Automatic role selection uses a fixed, compatible priority + The first configured technology in each row wins. An explicit nonblank role value bypasses this table. + + + + Cache + + Redis + + + local + + + + MessageBus + + RabbitMQ + + + Redis + + + local + + + + Queue + + AzureQueues + + + SQS + + + Redis + + + local + + + + Storage + + AzureStorage + → + + S3 + → + + Aliyun + → + + Folder + → + + local + + + Redis never supplies Storage + diff --git a/docs/public/assets/img/docs/redis-connection-ownership.svg b/docs/public/assets/img/docs/redis-connection-ownership.svg new file mode 100644 index 0000000000..73a707f1b2 --- /dev/null +++ b/docs/public/assets/img/docs/redis-connection-ownership.svg @@ -0,0 +1,70 @@ + + Redis connection ownership by exact effective connection string + In this example Cache and MessageBus resolve to endpoint A while Queue resolves to endpoint B. The Redis connection registry creates one shared connection for endpoint A and one isolated connection for endpoint B. The compatibility IConnectionMultiplexer and WebSocket mapping explicitly use the Cache connection. + + + + + + + + Redis connections are keyed by the exact effective string + Example: equal endpoints share one connection; different role endpoints remain isolated. + + + Cache + effective string A + + + MessageBus + effective string A + + + Queue + effective string B + + + RedisConnectionRegistry + lazy lookup by the complete, + exact effective connection string + owns connection disposal + + + + + + + Redis connection A + shared once by Cache + and MessageBus + + + Redis connection B + isolated Queue endpoint + even when Cache differs + + + + + + compatibility + IConnection + Multiplexer + + WebSocket + mapping + + + + + WebSocket mapping and IConnectionMultiplexer always follow Cache, never another Redis-backed role. + diff --git a/k8s/exceptionless/templates/elasticsearch.yaml b/k8s/exceptionless/templates/elasticsearch.yaml index eabd78145d..c92e80e262 100644 --- a/k8s/exceptionless/templates/elasticsearch.yaml +++ b/k8s/exceptionless/templates/elasticsearch.yaml @@ -29,11 +29,8 @@ spec: labels: app: {{ template "exceptionless.name" . }} component: {{ template "exceptionless.fullname" . }}-elasticsearch - chart: {{ template "exceptionless.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} spec: priorityClassName: high-priority initContainers: diff --git a/k8s/exceptionless/templates/redis.yaml b/k8s/exceptionless/templates/redis.yaml index d04ec02a41..fb58a96f36 100644 --- a/k8s/exceptionless/templates/redis.yaml +++ b/k8s/exceptionless/templates/redis.yaml @@ -19,11 +19,8 @@ spec: labels: app: {{ template "exceptionless.name" . }} component: {{ template "exceptionless.fullname" . }}-redis - chart: {{ template "exceptionless.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} spec: containers: - name: {{ template "exceptionless.name" . }}-redis diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index cfdd12902e..681c6fcb6e 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -262,16 +262,16 @@ public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions if (!logger.IsEnabled(LogLevel.Warning)) return; - if (String.IsNullOrEmpty(appOptions.CacheOptions.Provider)) + if (!IsDistributedProvider(appOptions.CacheOptions.Provider)) logger.LogWarning("Distributed cache is NOT enabled on {MachineName}", Environment.MachineName); - if (String.IsNullOrEmpty(appOptions.MessageBusOptions.Provider)) + if (!IsDistributedProvider(appOptions.MessageBusOptions.Provider)) logger.LogWarning("Distributed message bus is NOT enabled on {MachineName}", Environment.MachineName); - if (String.IsNullOrEmpty(appOptions.QueueOptions.Provider)) + if (!IsDistributedProvider(appOptions.QueueOptions.Provider)) logger.LogWarning("Distributed queue is NOT enabled on {MachineName}", Environment.MachineName); - if (String.IsNullOrEmpty(appOptions.StorageOptions.Provider)) + if (!IsDistributedProvider(appOptions.StorageOptions.Provider)) logger.LogWarning("Distributed storage is NOT enabled on {MachineName}", Environment.MachineName); if (!appOptions.EnableWebSockets) @@ -333,6 +333,12 @@ private static void LogConfigurationSummary(IServiceProvider serviceProvider, Ap GetEnabledIntegrations(options)); } + private static bool IsDistributedProvider(string? provider) + { + return !String.IsNullOrWhiteSpace(provider) + && !String.Equals(provider, "local", StringComparison.OrdinalIgnoreCase); + } + private static string GetProvider(string? provider) => String.IsNullOrWhiteSpace(provider) ? "disabled" : provider; diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index 99da0ce9d3..49b9fde952 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -82,6 +82,11 @@ public class AppOptions public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!; public SourceMapOptions SourceMapOptions { get; internal set; } = null!; + internal bool UsesRedis() => + String.Equals(CacheOptions.Provider, "redis", StringComparison.OrdinalIgnoreCase) + || String.Equals(MessageBusOptions.Provider, "redis", StringComparison.OrdinalIgnoreCase) + || String.Equals(QueueOptions.Provider, "redis", StringComparison.OrdinalIgnoreCase); + public static AppOptions ReadFromConfiguration(IConfiguration config) { var options = new AppOptions(); diff --git a/src/Exceptionless.Core/Configuration/CacheOptions.cs b/src/Exceptionless.Core/Configuration/CacheOptions.cs index 14eb53ba42..3343ec76fd 100644 --- a/src/Exceptionless.Core/Configuration/CacheOptions.cs +++ b/src/Exceptionless.Core/Configuration/CacheOptions.cs @@ -1,6 +1,4 @@ -using Exceptionless.Core.Extensions; -using Foundatio.Utility; -using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration; namespace Exceptionless.Core.Configuration; @@ -8,7 +6,7 @@ public class CacheOptions { public string? ConnectionString { get; internal set; } public string? Provider { get; internal set; } - public Dictionary Data { get; internal set; } = null!; + public Dictionary Data { get; internal set; } = new(StringComparer.OrdinalIgnoreCase); public string Scope { get; internal set; } = null!; public string ScopePrefix { get; internal set; } = null!; @@ -18,28 +16,10 @@ public static CacheOptions ReadFromConfiguration(IConfiguration config, AppOptio var options = new CacheOptions { Scope = appOptions.AppScope }; options.ScopePrefix = !String.IsNullOrEmpty(options.Scope) ? $"{options.Scope}-" : String.Empty; - string? cs = config.GetConnectionString("Cache"); - if (cs != null) - { - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); - string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; - - var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); - options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); - options.Data.AddRange(providerOptions); - - options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); - } - else - { - string? redisConnectionString = config.GetConnectionString("Redis"); - if (!String.IsNullOrEmpty(redisConnectionString)) - { - options.Provider = "redis"; - options.ConnectionString = redisConnectionString; - } - } + var providerConfiguration = ProviderConfigurationResolver.Resolve(config, ProviderRole.Cache); + options.Data = providerConfiguration.Data; + options.Provider = providerConfiguration.Provider; + options.ConnectionString = providerConfiguration.ConnectionString; return options; } diff --git a/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs b/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs index 31a3def6da..00e812653e 100644 --- a/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs +++ b/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs @@ -28,29 +28,33 @@ internal void Load(IDictionary envVariables) { var data = new Dictionary(StringComparer.OrdinalIgnoreCase); - IDictionaryEnumerator e = envVariables.GetEnumerator(); - try - { - while (e.MoveNext()) - { - string key = (string)e.Entry.Key; - string? value = (string?)e.Entry.Value; - - string normalizedKey = Normalize(key); - // remove EX_ prefix - if (normalizedKey.StartsWith("EX_")) - data[normalizedKey.Substring(3)] = value; - else - data[normalizedKey] = value; - } - } - finally - { - (e as IDisposable)?.Dispose(); - } + var variables = envVariables.Cast() + .Select(entry => new KeyValuePair((string)entry.Key, (string?)entry.Value)) + .ToList(); + + // Aspire-style variables are the base. Product-specific EX_ variables are + // applied second so their precedence never depends on dictionary iteration order. + AddVariables(data, variables, prefixed: false); + AddVariables(data, variables, prefixed: true); Data = data; } + private static void AddVariables( + IDictionary data, + IEnumerable> variables, + bool prefixed) + { + foreach ((string key, string? value) in variables) + { + string normalizedKey = Normalize(key); + bool hasPrefix = normalizedKey.StartsWith("EX_", StringComparison.OrdinalIgnoreCase); + if (hasPrefix != prefixed) + continue; + + data[hasPrefix ? normalizedKey[3..] : normalizedKey] = value; + } + } + private static string Normalize(string key) => key.Replace("__", ConfigurationPath.KeyDelimiter); } diff --git a/src/Exceptionless.Core/Configuration/MessageBusOptions.cs b/src/Exceptionless.Core/Configuration/MessageBusOptions.cs index a0a668c54a..f2808d7e34 100644 --- a/src/Exceptionless.Core/Configuration/MessageBusOptions.cs +++ b/src/Exceptionless.Core/Configuration/MessageBusOptions.cs @@ -1,6 +1,4 @@ -using Exceptionless.Core.Extensions; -using Foundatio.Utility; -using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration; namespace Exceptionless.Core.Configuration; @@ -8,7 +6,7 @@ public class MessageBusOptions { public string? ConnectionString { get; internal set; } public string? Provider { get; internal set; } - public Dictionary Data { get; internal set; } = null!; + public Dictionary Data { get; internal set; } = new(StringComparer.OrdinalIgnoreCase); public string Scope { get; internal set; } = null!; public string ScopePrefix { get; internal set; } = null!; @@ -20,30 +18,10 @@ public static MessageBusOptions ReadFromConfiguration(IConfiguration config, App options.ScopePrefix = !String.IsNullOrEmpty(options.Scope) ? $"{options.Scope}-" : String.Empty; options.Topic = config.GetValue(nameof(options.Topic), $"{options.ScopePrefix}messages"); - string? cs = config.GetConnectionString("MessageBus"); - - if (cs != null) - { - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); - string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; - - var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); - options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); - options.Data.AddRange(providerOptions); - - options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); - } - else - { - string? redisConnectionString = config.GetConnectionString("Redis"); - - if (!String.IsNullOrEmpty(redisConnectionString)) - { - options.Provider = "redis"; - options.ConnectionString = redisConnectionString; - } - } + var providerConfiguration = ProviderConfigurationResolver.Resolve(config, ProviderRole.MessageBus); + options.Data = providerConfiguration.Data; + options.Provider = providerConfiguration.Provider; + options.ConnectionString = providerConfiguration.ConnectionString; return options; } diff --git a/src/Exceptionless.Core/Configuration/ProviderConfigurationResolver.cs b/src/Exceptionless.Core/Configuration/ProviderConfigurationResolver.cs new file mode 100644 index 0000000000..ae1c21d2a4 --- /dev/null +++ b/src/Exceptionless.Core/Configuration/ProviderConfigurationResolver.cs @@ -0,0 +1,357 @@ +using Exceptionless.Core.Extensions; +using Foundatio.Utility; +using Microsoft.Extensions.Configuration; + +namespace Exceptionless.Core.Configuration; + +internal enum ProviderRole +{ + Cache, + MessageBus, + Queue, + Storage +} + +internal sealed record ProviderConfiguration( + string Provider, + string? ConnectionString, + Dictionary Data); + +internal static class ProviderConfigurationResolver +{ + private const string LocalProvider = "local"; + private const string ProviderKey = "provider"; + private const string RabbitMqProvider = "rabbitmq"; + private const string RedisProvider = "redis"; + private const string ServerKey = "server"; + + private enum ProviderConnectionStringFormat + { + KeyValue, + Redis, + AmqpUri + } + + private sealed record ProviderCandidate( + string Provider, + string ConnectionStringName, + ProviderConnectionStringFormat Format = ProviderConnectionStringFormat.KeyValue, + bool AllowsEmptyConfiguration = false); + + private static readonly IReadOnlyDictionary _providerCandidates = + new Dictionary + { + [ProviderRole.Cache] = + [ + new(RedisProvider, "Redis", ProviderConnectionStringFormat.Redis) + ], + [ProviderRole.MessageBus] = + [ + new(RabbitMqProvider, "RabbitMQ", ProviderConnectionStringFormat.AmqpUri), + new(RedisProvider, "Redis", ProviderConnectionStringFormat.Redis) + ], + [ProviderRole.Queue] = + [ + new("azurestorage", "AzureQueues"), + new("sqs", "SQS"), + new(RedisProvider, "Redis", ProviderConnectionStringFormat.Redis) + ], + [ProviderRole.Storage] = + [ + new("azurestorage", "AzureStorage"), + new("s3", "S3"), + new("aliyun", "Aliyun"), + new("folder", "Folder", AllowsEmptyConfiguration: true) + ] + }; + + public static ProviderConfiguration Resolve(IConfiguration configuration, ProviderRole role) + { + string roleName = role.ToString(); + string? selector = configuration.GetConnectionString(roleName); + if (String.IsNullOrWhiteSpace(selector)) + return ResolveInferred(configuration, role); + + selector = selector.Trim(); + if (String.Equals(selector, LocalProvider, StringComparison.OrdinalIgnoreCase)) + return CreateLocalConfiguration(); + + Dictionary roleData = new(StringComparer.OrdinalIgnoreCase); + string? inlineConnectionString = null; + try + { + roleData.AddRange(selector.ParseConnectionString()); + } + catch (ArgumentException) + { + if (!TryParseInlineConnectionString(selector, roleData, out inlineConnectionString)) + throw CreateInvalidConfigurationException(roleName); + } + + string? provider = roleData.GetString(ProviderKey); + if (String.IsNullOrWhiteSpace(provider)) + throw CreateInvalidConfigurationException(roleName); + + provider = provider.Trim().ToLowerInvariant(); + roleData[ProviderKey] = provider; + if (String.Equals(provider, RedisProvider, StringComparison.Ordinal) + && inlineConnectionString is null + && TryGetInlineRedisConnectionString(selector, roleData, out inlineConnectionString)) + { + roleData.Clear(); + roleData[ProviderKey] = provider; + } + + if (String.Equals(provider, LocalProvider, StringComparison.Ordinal)) + { + if (roleData.Count > 1 || inlineConnectionString is not null) + throw CreateInvalidConfigurationException(roleName); + + return CreateLocalConfiguration(); + } + + ProviderCandidate candidate = GetCandidate(role, provider); + if (inlineConnectionString is not null) + { + if (candidate.Format is ProviderConnectionStringFormat.KeyValue) + throw CreateInvalidConfigurationException(roleName); + + return CreateRawConfiguration(roleName, candidate, roleData, inlineConnectionString); + } + + string? providerConnectionString = GetProviderConnectionString(configuration, candidate); + Dictionary explicitData = roleData + .Where(pair => !String.Equals(pair.Key, ProviderKey, StringComparison.OrdinalIgnoreCase)) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.OrdinalIgnoreCase); + + if (candidate.Format is not ProviderConnectionStringFormat.KeyValue) + return ResolveRawConfiguration(roleName, candidate, roleData, explicitData, providerConnectionString); + + if (explicitData.Count == 0 && !String.IsNullOrWhiteSpace(providerConnectionString)) + return CreateConfiguration(candidate, providerConnectionString); + + var data = ParseProviderData(candidate, providerConnectionString); + data.AddRange(explicitData); + ValidateProviderIdentity(candidate, data); + data[ProviderKey] = provider; + + string? connectionString = data.BuildConnectionString([ProviderKey]); + if (String.IsNullOrWhiteSpace(connectionString)) + { + if (candidate.AllowsEmptyConfiguration) + return new ProviderConfiguration(provider, null, data); + + throw CreateInvalidConfigurationException(roleName); + } + + return new ProviderConfiguration(provider, connectionString, data); + } + + private static ProviderConfiguration ResolveInferred(IConfiguration configuration, ProviderRole role) + { + foreach (ProviderCandidate candidate in _providerCandidates[role]) + { + string? connectionString = configuration.GetConnectionString(candidate.ConnectionStringName); + if (String.IsNullOrWhiteSpace(connectionString)) + continue; + + return candidate.Format is not ProviderConnectionStringFormat.KeyValue + ? CreateRawConfiguration(role.ToString(), candidate, new Dictionary(StringComparer.OrdinalIgnoreCase), connectionString) + : CreateConfiguration(candidate, connectionString!); + } + + return CreateLocalConfiguration(); + } + + private static ProviderCandidate GetCandidate(ProviderRole role, string provider) + { + ProviderCandidate? candidate = _providerCandidates[role] + .FirstOrDefault(candidate => String.Equals(candidate.Provider, provider, StringComparison.OrdinalIgnoreCase)); + if (candidate is null) + throw new InvalidOperationException($"Provider '{provider}' is not supported for ConnectionStrings:{role}."); + + return candidate; + } + + private static string? GetProviderConnectionString(IConfiguration configuration, ProviderCandidate candidate) + { + string? connectionString = configuration.GetConnectionString(candidate.ConnectionStringName); + if (!String.IsNullOrWhiteSpace(connectionString)) + return connectionString; + + if (!String.Equals(candidate.ConnectionStringName, candidate.Provider, StringComparison.OrdinalIgnoreCase)) + connectionString = configuration.GetConnectionString(candidate.Provider); + + return connectionString; + } + + private static ProviderConfiguration CreateConfiguration(ProviderCandidate candidate, string connectionString) + { + connectionString = TrimMatchingQuotes(connectionString.Trim()); + var data = ParseProviderData(candidate, connectionString); + ValidateProviderIdentity(candidate, data); + data[ProviderKey] = candidate.Provider; + return new ProviderConfiguration(candidate.Provider, data.BuildConnectionString([ProviderKey]), data); + } + + private static ProviderConfiguration ResolveRawConfiguration( + string roleName, + ProviderCandidate candidate, + Dictionary roleData, + Dictionary explicitData, + string? providerConnectionString) + { + if (explicitData.Count == 1 + && explicitData.TryGetValue(ServerKey, out string? server) + && !String.IsNullOrWhiteSpace(server)) + { + return CreateRawConfiguration(roleName, candidate, roleData, server); + } + + if (explicitData.Count > 0 || String.IsNullOrWhiteSpace(providerConnectionString)) + throw CreateInvalidConfigurationException(roleName); + + return CreateRawConfiguration(roleName, candidate, roleData, providerConnectionString); + } + + private static ProviderConfiguration CreateRawConfiguration( + string roleName, + ProviderCandidate candidate, + Dictionary data, + string connectionString) + { + connectionString = TrimMatchingQuotes(connectionString.Trim()); + if (candidate.Format is ProviderConnectionStringFormat.AmqpUri && !IsSupportedAbsoluteUri(connectionString)) + throw CreateInvalidConfigurationException(roleName); + if (candidate.Format is ProviderConnectionStringFormat.Redis && ContainsProviderMetadata(connectionString)) + throw CreateInvalidConfigurationException(roleName); + + ValidateProviderIdentity(candidate, data); + data[ProviderKey] = candidate.Provider; + data[ServerKey] = connectionString; + return new ProviderConfiguration(candidate.Provider, connectionString, data); + } + + private static void ValidateProviderIdentity(ProviderCandidate candidate, IDictionary data) + { + string? configuredProvider = data.GetString(ProviderKey); + if (!String.IsNullOrWhiteSpace(configuredProvider) + && !String.Equals(configuredProvider, candidate.Provider, StringComparison.OrdinalIgnoreCase)) + { + throw CreateInvalidConfigurationException(candidate.ConnectionStringName); + } + } + + private static Dictionary ParseProviderData(ProviderCandidate candidate, string? connectionString) + { + if (String.IsNullOrWhiteSpace(connectionString)) + return new Dictionary(StringComparer.OrdinalIgnoreCase); + + try + { + return connectionString.ParseConnectionString(); + } + catch (ArgumentException) + { + throw CreateInvalidConfigurationException(candidate.ConnectionStringName); + } + } + + private static bool ContainsProviderMetadata(string connectionString) + { + try + { + return connectionString.ParseConnectionString().ContainsKey(ProviderKey); + } + catch (ArgumentException) + { + return false; + } + } + + private static bool TryGetInlineRedisConnectionString( + string selector, + IDictionary roleData, + out string? connectionString) + { + connectionString = null; + if (roleData.ContainsKey(ServerKey)) + return false; + + int separatorIndex = selector.IndexOf(';'); + if (separatorIndex < 0) + return false; + + string value = TrimMatchingQuotes(selector[(separatorIndex + 1)..].Trim()); + if (String.IsNullOrWhiteSpace(value) + || (value.Contains('=') && !value.Contains(','))) + return false; + + connectionString = value; + return true; + } + + private static bool TryParseInlineConnectionString( + string selector, + Dictionary data, + out string? connectionString) + { + connectionString = null; + int separatorIndex = selector.IndexOf(';'); + if (separatorIndex < 0) + return false; + + Dictionary providerData; + try + { + providerData = selector[..separatorIndex].ParseConnectionString(); + } + catch (ArgumentException) + { + return false; + } + + if (providerData.Count != 1 || String.IsNullOrWhiteSpace(providerData.GetString(ProviderKey))) + return false; + + string configuredConnectionString = TrimMatchingQuotes(selector[(separatorIndex + 1)..].Trim()); + if (String.IsNullOrWhiteSpace(configuredConnectionString)) + return false; + + data.AddRange(providerData); + connectionString = configuredConnectionString; + return true; + } + + private static ProviderConfiguration CreateLocalConfiguration() + { + return new ProviderConfiguration( + LocalProvider, + null, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [ProviderKey] = LocalProvider + }); + } + + private static bool IsSupportedAbsoluteUri(string value) + { + return Uri.TryCreate(value, UriKind.Absolute, out Uri? uri) + && (String.Equals(uri.Scheme, "amqp", StringComparison.OrdinalIgnoreCase) + || String.Equals(uri.Scheme, "amqps", StringComparison.OrdinalIgnoreCase)); + } + + private static string TrimMatchingQuotes(string value) + { + if (value.Length >= 2 && ((value[0] == '"' && value[^1] == '"') || (value[0] == '\'' && value[^1] == '\''))) + return value[1..^1]; + + return value; + } + + private static InvalidOperationException CreateInvalidConfigurationException(string connectionStringName) + { + return new InvalidOperationException( + $"ConnectionStrings:{connectionStringName} must specify a supported provider and a valid connection string, or use 'local'."); + } +} diff --git a/src/Exceptionless.Core/Configuration/QueueOptions.cs b/src/Exceptionless.Core/Configuration/QueueOptions.cs index 62c1a9e4a3..061c186e92 100644 --- a/src/Exceptionless.Core/Configuration/QueueOptions.cs +++ b/src/Exceptionless.Core/Configuration/QueueOptions.cs @@ -1,6 +1,4 @@ -using Exceptionless.Core.Extensions; -using Foundatio.Utility; -using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration; namespace Exceptionless.Core.Configuration; @@ -24,38 +22,10 @@ public static QueueOptions ReadFromConfiguration(IConfiguration config, AppOptio MetricsPollingInterval = appOptions.AppMode == AppMode.Development ? TimeSpan.FromSeconds(15) : TimeSpan.FromSeconds(5) }; - string? cs = config.GetConnectionString("Queue"); - if (!String.IsNullOrWhiteSpace(cs)) - { - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); - } - else - { - string? azureStorageConnectionString = config.GetConnectionString("AzureQueues"); - if (!String.IsNullOrEmpty(azureStorageConnectionString)) - { - options.Provider = "azurestorage"; - options.ConnectionString = azureStorageConnectionString; - options.Data = azureStorageConnectionString.ParseConnectionString(); - return options; - } - - string? redisConnectionString = config.GetConnectionString("Redis"); - if (!String.IsNullOrEmpty(redisConnectionString)) - { - options.Provider = "redis"; - options.ConnectionString = redisConnectionString; - options.Data = redisConnectionString.ParseConnectionString(); - return options; - } - } - - string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; - if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); - - options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); + var providerConfiguration = ProviderConfigurationResolver.Resolve(config, ProviderRole.Queue); + options.Data = providerConfiguration.Data; + options.Provider = providerConfiguration.Provider; + options.ConnectionString = providerConfiguration.ConnectionString; return options; } } diff --git a/src/Exceptionless.Core/Configuration/StorageOptions.cs b/src/Exceptionless.Core/Configuration/StorageOptions.cs index 085a459c47..b2347f6f99 100644 --- a/src/Exceptionless.Core/Configuration/StorageOptions.cs +++ b/src/Exceptionless.Core/Configuration/StorageOptions.cs @@ -1,6 +1,4 @@ -using Exceptionless.Core.Extensions; -using Foundatio.Utility; -using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration; namespace Exceptionless.Core.Configuration; @@ -21,29 +19,10 @@ public static StorageOptions ReadFromConfiguration(IConfiguration config, AppOpt ScopePrefix = !String.IsNullOrEmpty(appOptions.AppScope) ? $"{appOptions.AppScope}-" : String.Empty }; - string? cs = config.GetConnectionString("Storage"); - if (!String.IsNullOrWhiteSpace(cs)) - { - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); - } - else - { - string? azureStorageConnectionString = config.GetConnectionString("AzureStorage"); - if (!String.IsNullOrEmpty(azureStorageConnectionString)) - { - options.Provider = "azurestorage"; - options.ConnectionString = azureStorageConnectionString; - options.Data = azureStorageConnectionString.ParseConnectionString(); - return options; - } - } - - string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; - if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); - - options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); + var providerConfiguration = ProviderConfigurationResolver.Resolve(config, ProviderRole.Storage); + options.Data = providerConfiguration.Data; + options.Provider = providerConfiguration.Provider; + options.ConnectionString = providerConfiguration.ConnectionString; return options; } } diff --git a/src/Exceptionless.Core/Properties/AssemblyInfo.cs b/src/Exceptionless.Core/Properties/AssemblyInfo.cs index 2941889e70..96b5cdf649 100644 --- a/src/Exceptionless.Core/Properties/AssemblyInfo.cs +++ b/src/Exceptionless.Core/Properties/AssemblyInfo.cs @@ -3,3 +3,4 @@ [assembly: InternalsVisibleTo("Exceptionless.Tests")] [assembly: InternalsVisibleTo("Exceptionless.Web")] [assembly: InternalsVisibleTo("Exceptionless.Insulation")] +[assembly: InternalsVisibleTo("Exceptionless.Job")] diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index 5ce4f4f7b3..e60f7bc7ef 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -62,6 +62,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO if (!String.IsNullOrEmpty(appOptions.MaxMindGeoIpKey)) services.ReplaceSingleton(); + if (appOptions.UsesRedis()) + { + ValidateRedisConnectionStrings(appOptions); + services.AddSingleton(); + } + RegisterCache(services, appOptions.CacheOptions); RegisterMessageBus(services, appOptions.MessageBusOptions); RegisterQueue(services, appOptions.QueueOptions, runMaintenanceTasks); @@ -73,6 +79,43 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.ReplaceSingleton(); } + private static void ValidateRedisConnectionStrings(AppOptions options) + { + var roles = new (string Name, string? Provider, string? ConnectionString)[] + { + ("Cache", options.CacheOptions.Provider, options.CacheOptions.ConnectionString), + ("MessageBus", options.MessageBusOptions.Provider, options.MessageBusOptions.ConnectionString), + ("Queue", options.QueueOptions.Provider, options.QueueOptions.ConnectionString) + }; + var validated = new HashSet(StringComparer.Ordinal); + + foreach ((string name, string? provider, string? connectionString) in roles) + { + if (!String.Equals(provider, "redis", StringComparison.OrdinalIgnoreCase)) + continue; + + if (String.IsNullOrWhiteSpace(connectionString)) + throw CreateInvalidRedisConfigurationException(name); + + if (!validated.Add(connectionString)) + continue; + + try + { + ConfigurationOptions configuration = ConfigurationOptions.Parse(connectionString); + if (configuration.EndPoints.Count == 0) + throw CreateInvalidRedisConfigurationException(name); + } + catch (ArgumentException) + { + throw CreateInvalidRedisConfigurationException(name); + } + } + } + + private static InvalidOperationException CreateInvalidRedisConfigurationException(string role) + => new($"The Redis connection string selected for {role} is invalid."); + private static IHealthChecksBuilder RegisterHealthChecks(IServiceCollection services) { services.AddStartupActionToWaitForHealthChecks("Critical"); @@ -104,14 +147,16 @@ private static void RegisterCache(IServiceCollection container, CacheOptions opt { if (String.Equals(options.Provider, "redis")) { - container.ReplaceSingleton(s => GetRedisConnection(options.ConnectionString!, s.GetRequiredService())); + container.ReplaceSingleton(s => + s.GetRequiredService().GetCacheConnection(options.ConnectionString!)); if (!String.IsNullOrEmpty(options.Scope)) container.ReplaceSingleton(s => new ScopedCacheClient(CreateRedisCacheClient(s), options.Scope)); else container.ReplaceSingleton(CreateRedisCacheClient); - container.ReplaceSingleton(); + container.ReplaceSingleton(s => new RedisConnectionMapping( + s.GetRequiredService().GetConnection(options.ConnectionString!))); } } @@ -119,11 +164,9 @@ private static void RegisterMessageBus(IServiceCollection container, MessageBusO { if (String.Equals(options.Provider, "redis")) { - container.ReplaceSingleton(s => GetRedisConnection(options.ConnectionString!, s.GetRequiredService())); - container.ReplaceSingleton(s => new RedisMessageBus(new RedisMessageBusOptions { - Subscriber = s.GetRequiredService().GetSubscriber(), + Subscriber = s.GetRequiredService().GetConnection(options.ConnectionString!).GetSubscriber(), Topic = options.Topic, Serializer = s.GetRequiredService(), TimeProvider = s.GetRequiredService(), @@ -145,11 +188,6 @@ private static void RegisterMessageBus(IServiceCollection container, MessageBusO } } - private static IConnectionMultiplexer GetRedisConnection(string connectionString, ILoggerFactory loggerFactory) - { - return ConnectionMultiplexer.Connect(connectionString, o => o.LoggerFactory = loggerFactory); - } - private static void RegisterQueue(IServiceCollection container, QueueOptions options, bool runMaintenanceTasks) { if (String.Equals(options.Provider, "azurestorage")) @@ -262,7 +300,7 @@ private static IQueue CreateRedisQueue(IServiceProvider container, QueueOp { return new RedisQueue(new RedisQueueOptions { - ConnectionMultiplexer = container.GetRequiredService(), + ConnectionMultiplexer = container.GetRequiredService().GetConnection(options.ConnectionString!), Name = GetQueueName(options), Retries = retries, Behaviors = container.GetServices>().ToList(), diff --git a/src/Exceptionless.Insulation/Properties/AssemblyInfo.cs b/src/Exceptionless.Insulation/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..5be2f7fe08 --- /dev/null +++ b/src/Exceptionless.Insulation/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Exceptionless.Tests")] diff --git a/src/Exceptionless.Insulation/Redis/RedisConnectionRegistry.cs b/src/Exceptionless.Insulation/Redis/RedisConnectionRegistry.cs new file mode 100644 index 0000000000..5eaa2fd96e --- /dev/null +++ b/src/Exceptionless.Insulation/Redis/RedisConnectionRegistry.cs @@ -0,0 +1,87 @@ +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace Exceptionless.Insulation.Redis; + +internal sealed class RedisConnectionRegistry : IDisposable +{ + private readonly Dictionary> _connections = new(StringComparer.Ordinal); + private readonly HashSet _externallyOwnedConnections = new(ReferenceEqualityComparer.Instance); + private readonly Func _connectionFactory; + private readonly ILoggerFactory _loggerFactory; + private readonly Lock _lock = new(); + private bool _disposed; + + public RedisConnectionRegistry(ILoggerFactory loggerFactory) + : this(loggerFactory, static (connectionString, factory) => + ConnectionMultiplexer.Connect(connectionString, options => options.LoggerFactory = factory)) + { + } + + internal RedisConnectionRegistry( + ILoggerFactory loggerFactory, + Func connectionFactory) + { + _loggerFactory = loggerFactory; + _connectionFactory = connectionFactory; + } + + public IConnectionMultiplexer GetConnection(string connectionString) + => GetConnection(connectionString, externallyOwned: false); + + public IConnectionMultiplexer GetCacheConnection(string connectionString) + => GetConnection(connectionString, externallyOwned: true); + + private IConnectionMultiplexer GetConnection(string connectionString, bool externallyOwned) + { + if (String.IsNullOrWhiteSpace(connectionString)) + throw new ArgumentException("A Redis connection string is required.", nameof(connectionString)); + + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_connections.TryGetValue(connectionString, out Lazy? connection)) + { + connection = new Lazy( + () => _connectionFactory(connectionString, _loggerFactory), + LazyThreadSafetyMode.ExecutionAndPublication); + _connections.Add(connectionString, connection); + } + + try + { + IConnectionMultiplexer multiplexer = connection.Value; + if (externallyOwned) + _externallyOwnedConnections.Add(multiplexer); + + return multiplexer; + } + catch + { + _connections.Remove(connectionString); + throw; + } + } + } + + public void Dispose() + { + List connectionsToDispose; + lock (_lock) + { + if (_disposed) + return; + + _disposed = true; + connectionsToDispose = _connections.Values + .Where(connection => connection.IsValueCreated) + .Select(connection => connection.Value) + .Distinct((IEqualityComparer)ReferenceEqualityComparer.Instance) + .Where(connection => !_externallyOwnedConnections.Contains(connection)) + .ToList(); + } + + foreach (IConnectionMultiplexer multiplexer in connectionsToDispose) + multiplexer.Dispose(); + } +} diff --git a/src/Exceptionless.Job/Program.cs b/src/Exceptionless.Job/Program.cs index 773bf8f3c9..6cc3810c76 100644 --- a/src/Exceptionless.Job/Program.cs +++ b/src/Exceptionless.Job/Program.cs @@ -65,7 +65,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) // only poll the queue metrics if this process is going to run the stack event count job options.QueueOptions.MetricsPollingEnabled = jobOptions.StackEventCount; - var apmConfig = new ApmConfig(config, $"job-{jobOptions.JobName.ToLowerUnderscoredWords('-')}", options.InformationalVersion, options.CacheOptions.Provider == "redis"); + var apmConfig = new ApmConfig(config, $"job-{jobOptions.JobName.ToLowerUnderscoredWords('-')}", options.InformationalVersion, options.UsesRedis()); Log.Information("Bootstrapping Exceptionless {JobName} job(s) in {AppMode} mode ({InformationalVersion}) on {MachineName} with scope {AppScope}", jobOptions.JobName ?? "All", environment, options.InformationalVersion, Environment.MachineName, options.AppScope); diff --git a/src/Exceptionless.Job/appsettings.Development.yml b/src/Exceptionless.Job/appsettings.Development.yml index 7b7a1b7601..5d2d52b745 100644 --- a/src/Exceptionless.Job/appsettings.Development.yml +++ b/src/Exceptionless.Job/appsettings.Development.yml @@ -2,10 +2,7 @@ ConnectionStrings: # Redis: localhost,abortConnect=false # Elasticsearch: server=https://elastic:elastic@localhost:9200 -# Cache: provider=redis; -# MessageBus: provider=redis; -# Queue: provider=redis; -# Storage: provider=folder;path=..\Exceptionless.Web\storage +# Folder: path=..\Exceptionless.Web\storage Email: smtp://localhost:1025 # Base url for the ui used to build links in emails and other places. diff --git a/src/Exceptionless.Job/appsettings.Production.yml b/src/Exceptionless.Job/appsettings.Production.yml index b01a5221e1..fda28540bf 100644 --- a/src/Exceptionless.Job/appsettings.Production.yml +++ b/src/Exceptionless.Job/appsettings.Production.yml @@ -1,10 +1,8 @@ --- ConnectionStrings: # Elasticsearch: server=http://localhost:9200 -# Cache: provider=redis;server="localhost,abortConnect=false" -# MessageBus: provider=redis;server="localhost,abortConnect=false" -# Queue: provider=redis;server="localhost,abortConnect=false" -# Storage: '' +# Redis: localhost,abortConnect=false +# Folder: path=..\Exceptionless.Web\storage # Email: 'smtps://user:password@domain.com:587' # LDAP: '' OAuth: FacebookId=395178683904310;GitHubId=7ef1dd5bfbc4ccf7f5ef;GoogleId=809763155066-enkkdmt4ierc33q9cft9nf5d5c02h30q.apps.googleusercontent.com;MicrosoftId=000000004C137E8B;SlackId=34500115540.177239122322; diff --git a/src/Exceptionless.Job/appsettings.Staging.yml b/src/Exceptionless.Job/appsettings.Staging.yml index db415b6211..e1d1fb7fa7 100644 --- a/src/Exceptionless.Job/appsettings.Staging.yml +++ b/src/Exceptionless.Job/appsettings.Staging.yml @@ -2,10 +2,7 @@ ConnectionStrings: # Redis: localhost,abortConnect=false # Elasticsearch: server=http://localhost:9200;replicas=0 -# Cache: provider=redis; -# MessageBus: provider=redis; -# Queue: provider=redis; -# Storage: provider=folder;path=.\storage= +# Folder: path=..\Exceptionless.Web\storage OAuth: FacebookId=395178683904310;GitHubId=7ef1dd5bfbc4ccf7f5ef;GoogleId=809763155066-enkkdmt4ierc33q9cft9nf5d5c02h30q.apps.googleusercontent.com;MicrosoftId=000000004C137E8B;SlackId=34500115540.177239122322; # Base url for the ui used to build links in emails and other places. diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index f80d46d4fb..725ec4db74 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -89,7 +89,7 @@ public static async Task Main(string[] args) var options = AppOptions.ReadFromConfiguration(configuration); options.QueueOptions.MetricsPollingEnabled = options.RunJobsInProcess; - var apmConfig = new ApmConfig(configuration, "web", options.InformationalVersion, options.CacheOptions.Provider == "redis"); + var apmConfig = new ApmConfig(configuration, "web", options.InformationalVersion, options.UsesRedis()); Log.Information("Bootstrapping Exceptionless Web in {AppMode} mode ({InformationalVersion}) on {MachineName} with scope {AppScope}", environment, options.InformationalVersion, Environment.MachineName, options.AppScope); diff --git a/src/Exceptionless.Web/appsettings.Development.yml b/src/Exceptionless.Web/appsettings.Development.yml index b2a91a34bd..735bf7bef1 100644 --- a/src/Exceptionless.Web/appsettings.Development.yml +++ b/src/Exceptionless.Web/appsettings.Development.yml @@ -2,10 +2,7 @@ ConnectionStrings: # Redis: localhost,abortConnect=false # Elasticsearch: server=https://elastic:elastic@localhost:9200 -# Cache: provider=redis; -# MessageBus: provider=redis; -# Queue: provider=redis; -# Storage: provider=folder;path=.\storage +# Folder: path=.\storage # LDAP: '' # Email: smtp://localhost:1025 diff --git a/src/Exceptionless.Web/appsettings.Production.yml b/src/Exceptionless.Web/appsettings.Production.yml index a6bbd82962..4647bc5e7a 100644 --- a/src/Exceptionless.Web/appsettings.Production.yml +++ b/src/Exceptionless.Web/appsettings.Production.yml @@ -1,10 +1,8 @@ --- ConnectionStrings: # Elasticsearch: server=http://localhost:9200 -# Cache: provider=redis;server="localhost,abortConnect=false" -# MessageBus: provider=redis;server="localhost,abortConnect=false" -# Queue: provider=redis;server="localhost,abortConnect=false" -# Storage: '' +# Redis: localhost,abortConnect=false +# Folder: path=.\storage # Email: 'smtps://user:password@domain.com:587' # LDAP: '' OAuth: FacebookId=395178683904310;GitHubId=7ef1dd5bfbc4ccf7f5ef;GoogleId=809763155066-enkkdmt4ierc33q9cft9nf5d5c02h30q.apps.googleusercontent.com;MicrosoftId=000000004C137E8B;SlackId=34500115540.177239122322; diff --git a/src/Exceptionless.Web/appsettings.Staging.yml b/src/Exceptionless.Web/appsettings.Staging.yml index 9d97c7b5a4..43b3068968 100644 --- a/src/Exceptionless.Web/appsettings.Staging.yml +++ b/src/Exceptionless.Web/appsettings.Staging.yml @@ -2,10 +2,7 @@ ConnectionStrings: # Redis: localhost,abortConnect=false # Elasticsearch: server=http://localhost:9200;replicas=0 -# Cache: provider=redis; -# MessageBus: provider=redis; -# Queue: provider=redis; -# Storage: provider=folder;path=.\storage= +# Folder: path=.\storage OAuth: FacebookId=395178683904310;GitHubId=7ef1dd5bfbc4ccf7f5ef;GoogleId=809763155066-enkkdmt4ierc33q9cft9nf5d5c02h30q.apps.googleusercontent.com;MicrosoftId=000000004C137E8B;SlackId=34500115540.177239122322; # Base url for the ui used to build links in emails and other places. diff --git a/tests/Exceptionless.Tests/BootstrapperTests.cs b/tests/Exceptionless.Tests/BootstrapperTests.cs index 188d72b06e..fad58d5ecd 100644 --- a/tests/Exceptionless.Tests/BootstrapperTests.cs +++ b/tests/Exceptionless.Tests/BootstrapperTests.cs @@ -60,10 +60,10 @@ public void LogConfiguration_DisabledServices_LogsDisabledSummaryAndPreservesWar Bootstrapper.LogConfiguration(serviceProvider, options, logger); string output = String.Join(Environment.NewLine, logger.Entries.Select(entry => entry.Message)); - Assert.Contains("cache disabled at not configured", output, StringComparison.Ordinal); - Assert.Contains("message bus disabled at not configured", output, StringComparison.Ordinal); - Assert.Contains("queue disabled at not configured", output, StringComparison.Ordinal); - Assert.Contains("storage disabled at not configured", output, StringComparison.Ordinal); + Assert.Contains("cache local at not configured", output, StringComparison.Ordinal); + Assert.Contains("message bus local at not configured", output, StringComparison.Ordinal); + Assert.Contains("queue local at not configured", output, StringComparison.Ordinal); + Assert.Contains("storage local at not configured", output, StringComparison.Ordinal); Assert.Contains("event submission disabled", output, StringComparison.Ordinal); Assert.Contains("WebSockets disabled", output, StringComparison.Ordinal); Assert.Contains("email disabled", output, StringComparison.Ordinal); diff --git a/tests/Exceptionless.Tests/Configuration/CustomEnvironmentVariablesConfigurationProviderTests.cs b/tests/Exceptionless.Tests/Configuration/CustomEnvironmentVariablesConfigurationProviderTests.cs new file mode 100644 index 0000000000..e39e5bc1de --- /dev/null +++ b/tests/Exceptionless.Tests/Configuration/CustomEnvironmentVariablesConfigurationProviderTests.cs @@ -0,0 +1,33 @@ +using System.Collections; +using Exceptionless.Core.Configuration; +using Xunit; + +namespace Exceptionless.Tests.Configuration; + +public class CustomEnvironmentVariablesConfigurationProviderTests +{ + [Fact] + public void Load_ExAndAspireVariablesNormalizeToSameKey_ExValueWinsRegardlessOfEnumerationOrder() + { + var first = new System.Collections.Specialized.OrderedDictionary + { + ["EX_ConnectionStrings__Redis"] = "ex:6379", + ["ConnectionStrings__Redis"] = "aspire:6379" + }; + var second = new System.Collections.Specialized.OrderedDictionary + { + ["ConnectionStrings__Redis"] = "aspire:6379", + ["EX_ConnectionStrings__Redis"] = "ex:6379" + }; + + var firstProvider = new CustomEnvironmentVariablesConfigurationProvider(); + firstProvider.Load(first); + var secondProvider = new CustomEnvironmentVariablesConfigurationProvider(); + secondProvider.Load(second); + + Assert.True(firstProvider.TryGet("ConnectionStrings:Redis", out string? firstValue)); + Assert.True(secondProvider.TryGet("ConnectionStrings:Redis", out string? secondValue)); + Assert.Equal("ex:6379", firstValue); + Assert.Equal("ex:6379", secondValue); + } +} diff --git a/tests/Exceptionless.Tests/Configuration/MessageBusOptionsTests.cs b/tests/Exceptionless.Tests/Configuration/MessageBusOptionsTests.cs new file mode 100644 index 0000000000..709f2b31a7 --- /dev/null +++ b/tests/Exceptionless.Tests/Configuration/MessageBusOptionsTests.cs @@ -0,0 +1,90 @@ +using Exceptionless.Core; +using Exceptionless.Core.Configuration; +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace Exceptionless.Tests.Configuration; + +public class MessageBusOptionsTests +{ + [Theory] + [InlineData("provider=rabbitmq;amqp://localhost/%2F", "amqp://localhost/%2F")] + [InlineData("provider=rabbitmq;\"amqp://localhost/%2F\"", "amqp://localhost/%2F")] + [InlineData("provider=rabbitmq;'amqp://localhost/%2F'", "amqp://localhost/%2F")] + [InlineData( + " PROVIDER = \"RABBITMQ\" ; 'amqps://user:p%40ss@rabbit.example.com:5671/team%2Fprod?heartbeat=30&connection_timeout=10000' ", + "amqps://user:p%40ss@rabbit.example.com:5671/team%2Fprod?heartbeat=30&connection_timeout=10000")] + public void ReadFromConfiguration_WithInlineRabbitMqUri_PreservesRawConnectionString(string configuredConnectionString, string expectedConnectionString) + { + var options = ReadOptions(new Dictionary + { + ["ConnectionStrings:MessageBus"] = configuredConnectionString + }); + + Assert.Equal("rabbitmq", options.Provider); + Assert.Equal(expectedConnectionString, options.ConnectionString); + Assert.Equal(expectedConnectionString, options.Data["server"]); + } + + [Theory] + [InlineData("provider=rabbitmq", "amqp://localhost/%2F", "amqp://localhost/%2F")] + [InlineData("provider=rabbitmq;", "'amqp://localhost/%2F'", "amqp://localhost/%2F")] + [InlineData( + "provider=RaBbItMq", + "\"amqps://user:p%40ss@rabbit.example.com:5671/team%2Fprod?heartbeat=30\"", + "amqps://user:p%40ss@rabbit.example.com:5671/team%2Fprod?heartbeat=30")] + public void ReadFromConfiguration_WithNamedRabbitMqUri_PreservesRawConnectionString(string selector, string configuredConnectionString, string expectedConnectionString) + { + var options = ReadOptions(new Dictionary + { + ["ConnectionStrings:MessageBus"] = selector, + ["ConnectionStrings:rabbitmq"] = configuredConnectionString + }); + + Assert.Equal("rabbitmq", options.Provider); + Assert.Equal(expectedConnectionString, options.ConnectionString); + Assert.Equal(expectedConnectionString, options.Data["server"]); + } + + [Fact] + public void ReadFromConfiguration_WithRedisProviderAndNamedNativeString_PreservesAtomicValue() + { + var options = ReadOptions(new Dictionary + { + ["ConnectionStrings:MessageBus"] = "provider=redis", + ["ConnectionStrings:redis"] = "ssl=true,localhost:6379,abortConnect=false" + }); + + Assert.Equal("redis", options.Provider); + Assert.Equal("ssl=true,localhost:6379,abortConnect=false", options.ConnectionString); + Assert.Equal("ssl=true,localhost:6379,abortConnect=false", options.Data["server"]); + } + + [Fact] + public void ReadFromConfiguration_WithLegacyRedisServerWrapper_ProducesNativeConnectionString() + { + var options = ReadOptions(new Dictionary + { + ["ConnectionStrings:MessageBus"] = "provider=redis;server=localhost:6379,abortConnect=false" + }); + + Assert.Equal("redis", options.Provider); + Assert.Equal("localhost:6379,abortConnect=false", options.ConnectionString); + Assert.Equal("localhost:6379,abortConnect=false", options.Data["server"]); + } + + private static MessageBusOptions ReadOptions(Dictionary values) + { + var configuration = CreateConfiguration(values); + var appOptions = new AppOptions { AppScope = "production" }; + + return MessageBusOptions.ReadFromConfiguration(configuration, appOptions); + } + + private static IConfiguration CreateConfiguration(Dictionary values) + { + return new ConfigurationBuilder() + .AddInMemoryCollection(values) + .Build(); + } +} diff --git a/tests/Exceptionless.Tests/Configuration/ProviderConfigurationTests.cs b/tests/Exceptionless.Tests/Configuration/ProviderConfigurationTests.cs new file mode 100644 index 0000000000..cfed0a85eb --- /dev/null +++ b/tests/Exceptionless.Tests/Configuration/ProviderConfigurationTests.cs @@ -0,0 +1,372 @@ +using Exceptionless.Core; +using Exceptionless.Core.Configuration; +using Microsoft.Extensions.Configuration; +using StackExchange.Redis; +using Xunit; + +namespace Exceptionless.Tests.Configuration; + +public class ProviderConfigurationTests +{ + [Theory] + [InlineData("redis:6379,abortConnect=false")] + [InlineData("redis,abortConnect=false")] + public void ReadFromConfiguration_ExistingHelmAndDockerConfiguration_PreservesExplicitProviders(string redisConnectionString) + { + IConfiguration configuration = CreateConfiguration(new() + { + ["ConnectionStrings:Redis"] = redisConnectionString, + ["ConnectionStrings:Cache"] = "provider=redis;", + ["ConnectionStrings:MessageBus"] = "provider=redis;", + ["ConnectionStrings:Queue"] = "provider=redis;", + ["ConnectionStrings:Storage"] = "provider=folder;path=/app/storage" + }); + + AppOptions options = ReadOptionsFromConfiguration(configuration); + + Assert.Equal("redis", options.CacheOptions.Provider); + Assert.Equal("redis", options.MessageBusOptions.Provider); + Assert.Equal("redis", options.QueueOptions.Provider); + Assert.Equal("folder", options.StorageOptions.Provider); + Assert.Equal("/app/storage", options.StorageOptions.Data["path"]); + } + + [Theory] + [InlineData("Cache", "Redis", "redis:6379", "redis")] + [InlineData("MessageBus", "RabbitMQ", "amqp://rabbit/%2F", "rabbitmq")] + [InlineData("MessageBus", "Redis", "redis:6379", "redis")] + [InlineData("Queue", "AzureQueues", "UseDevelopmentStorage=true", "azurestorage")] + [InlineData("Queue", "SQS", "region=us-east-2", "sqs")] + [InlineData("Queue", "Redis", "redis:6379", "redis")] + [InlineData("Storage", "AzureStorage", "UseDevelopmentStorage=true", "azurestorage")] + [InlineData("Storage", "S3", "bucket=events", "s3")] + [InlineData("Storage", "Aliyun", "bucket=events", "aliyun")] + [InlineData("Storage", "Folder", "path=/app/storage", "folder")] + public void Resolve_SingleCompatibleTechnology_InfersProvider( + string roleName, + string connectionStringName, + string connectionString, + string expectedProvider) + { + IConfiguration configuration = CreateConfiguration(new() + { + [$"ConnectionStrings:{connectionStringName}"] = connectionString + }); + + ProviderRole role = Enum.Parse(roleName); + ProviderConfiguration resolved = ProviderConfigurationResolver.Resolve(configuration, role); + + Assert.Equal(expectedProvider, resolved.Provider); + } + + [Fact] + public void ReadFromConfiguration_AspireConfiguration_InfersCompatibleProviders() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Redis"] = "localhost:6379", + ["ConnectionStrings:AzureStorage"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:AzureQueues"] = "UseDevelopmentStorage=true" + }); + + Assert.Equal("redis", options.CacheOptions.Provider); + Assert.Equal("redis", options.MessageBusOptions.Provider); + Assert.Equal("azurestorage", options.QueueOptions.Provider); + Assert.Equal("azurestorage", options.StorageOptions.Provider); + Assert.Equal("localhost:6379", options.CacheOptions.ConnectionString); + Assert.Equal("localhost:6379", options.MessageBusOptions.ConnectionString); + Assert.Equal("UseDevelopmentStorage=true", options.QueueOptions.ConnectionString); + Assert.Equal("UseDevelopmentStorage=true", options.StorageOptions.ConnectionString); + } + + [Fact] + public void ReadFromConfiguration_AllAutomaticCandidates_UsesFixedRolePriorities() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Redis"] = "redis:6379", + ["ConnectionStrings:RabbitMQ"] = "amqps://rabbit.example.test/%2F", + ["ConnectionStrings:AzureQueues"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:SQS"] = "region=us-east-2", + ["ConnectionStrings:AzureStorage"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:S3"] = "bucket=events", + ["ConnectionStrings:Aliyun"] = "bucket=events" + }); + + Assert.Equal("redis", options.CacheOptions.Provider); + Assert.Equal("rabbitmq", options.MessageBusOptions.Provider); + Assert.Equal("azurestorage", options.QueueOptions.Provider); + Assert.Equal("azurestorage", options.StorageOptions.Provider); + } + + [Fact] + public void ReadFromConfiguration_ExplicitRoles_OverrideAutomaticPriority() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Redis"] = "redis:6379", + ["ConnectionStrings:RabbitMQ"] = "amqp://rabbit.example.test/%2F", + ["ConnectionStrings:AzureQueues"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:MessageBus"] = "provider=redis", + ["ConnectionStrings:Queue"] = "provider=redis", + ["ConnectionStrings:Storage"] = "provider=folder;path=/data/events" + }); + + Assert.Equal("redis", options.MessageBusOptions.Provider); + Assert.Equal("redis", options.QueueOptions.Provider); + Assert.Equal("folder", options.StorageOptions.Provider); + } + + [Fact] + public void ReadFromConfiguration_LegacyRoleData_OverridesStructuredSharedProviderData() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:S3"] = "bucket=shared;region=us-east-1", + ["ConnectionStrings:Storage"] = "provider=s3;bucket=events" + }); + + Assert.Equal("events", options.StorageOptions.Data["bucket"]); + Assert.Equal("us-east-1", options.StorageOptions.Data["region"]); + } + + [Fact] + public void ReadFromConfiguration_OpaqueTechnologyWithLegacyRoleOptions_Throws() + { + var values = new Dictionary + { + ["ConnectionStrings:Redis"] = "redis:6379,abortConnect=false", + ["ConnectionStrings:Cache"] = "provider=redis;ssl=true" + }; + + Assert.Throws(() => ReadOptions(values)); + } + + [Fact] + public void ReadFromConfiguration_TechnologyConnectionWithMismatchedProvider_Throws() + { + var values = new Dictionary + { + ["ConnectionStrings:Redis"] = "provider=unknown;server=redis:6379" + }; + + Assert.Throws(() => ReadOptions(values)); + } + + [Theory] + [InlineData("Queue", "provider=azurestorage;https://queue.example.test")] + [InlineData("Queue", "provider=sqs;https://sqs.example.test")] + [InlineData("Storage", "provider=azurestorage;https://storage.example.test")] + [InlineData("Storage", "provider=s3;https://s3.example.test")] + public void Resolve_StructuredProviderWithOpaqueInlineValue_Throws(string roleName, string selector) + { + IConfiguration configuration = CreateConfiguration(new() + { + [$"ConnectionStrings:{roleName}"] = selector + }); + + ProviderRole role = Enum.Parse(roleName); + + Assert.Throws(() => ProviderConfigurationResolver.Resolve(configuration, role)); + } + + [Fact] + public void Resolve_RedisSelectedForStorage_Throws() + { + IConfiguration configuration = CreateConfiguration(new() + { + ["ConnectionStrings:Storage"] = "provider=redis;redis:6379" + }); + + Assert.Throws(() => ProviderConfigurationResolver.Resolve(configuration, ProviderRole.Storage)); + } + + [Fact] + public void ReadFromConfiguration_RedisOnly_NeverSelectsStorage() + { + AppOptions options = ReadOptions(new() { ["ConnectionStrings:Redis"] = "redis:6379" }); + + Assert.Equal("redis", options.CacheOptions.Provider); + Assert.Equal("redis", options.MessageBusOptions.Provider); + Assert.Equal("redis", options.QueueOptions.Provider); + Assert.Equal("local", options.StorageOptions.Provider); + Assert.True(options.UsesRedis()); + } + + [Fact] + public void ReadFromConfiguration_RedisQueueOnly_EnablesRedisTelemetry() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Cache"] = "local", + ["ConnectionStrings:MessageBus"] = "local", + ["ConnectionStrings:Queue"] = "provider=redis;queue:6379", + ["ConnectionStrings:Storage"] = "local" + }); + + Assert.True(options.UsesRedis()); + } + + [Fact] + public void ReadFromConfiguration_Local_PreventsInference() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Redis"] = "redis:6379", + ["ConnectionStrings:RabbitMQ"] = "amqp://rabbit.example.test/%2F", + ["ConnectionStrings:AzureQueues"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:AzureStorage"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:Cache"] = "local", + ["ConnectionStrings:MessageBus"] = "provider=LOCAL", + ["ConnectionStrings:Queue"] = "local", + ["ConnectionStrings:Storage"] = "local" + }); + + Assert.Equal("local", options.CacheOptions.Provider); + Assert.Equal("local", options.MessageBusOptions.Provider); + Assert.Equal("local", options.QueueOptions.Provider); + Assert.Equal("local", options.StorageOptions.Provider); + Assert.False(options.UsesRedis()); + } + + [Fact] + public void ReadFromConfiguration_BlankRoleValue_CountsAsAbsent() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Redis"] = "redis:6379", + ["ConnectionStrings:Cache"] = " " + }); + + Assert.Equal("redis", options.CacheOptions.Provider); + } + + [Theory] + [InlineData("provider=unknown")] + [InlineData("provider=redis")] + [InlineData("Redis")] + [InlineData("Redis;ssl=true")] + [InlineData("redis:6379")] + [InlineData("provider=local;server=redis:6379")] + public void ReadFromConfiguration_InvalidExplicitCache_Throws(string selector) + { + var values = new Dictionary { ["ConnectionStrings:Cache"] = selector }; + + Assert.Throws(() => ReadOptions(values)); + } + + [Theory] + [InlineData("provider=rabbitmq;amqp://localhost/%2F", "amqp://localhost/%2F")] + [InlineData("provider=rabbitmq;\"amqps://user:p%40ss@rabbit.example.test:5671/team%2Fprod?heartbeat=30\"", "amqps://user:p%40ss@rabbit.example.test:5671/team%2Fprod?heartbeat=30")] + [InlineData("provider=rabbitmq;server=\"amqps://rabbit.example.test/%2F\"", "amqps://rabbit.example.test/%2F")] + public void ReadFromConfiguration_LegacyAndInlineRabbitMq_PreservesRawUri(string selector, string expected) + { + AppOptions options = ReadOptions(new() { ["ConnectionStrings:MessageBus"] = selector }); + + Assert.Equal("rabbitmq", options.MessageBusOptions.Provider); + Assert.Equal(expected, options.MessageBusOptions.ConnectionString); + Assert.Equal(expected, options.MessageBusOptions.Data["server"]); + } + + [Fact] + public void ReadFromConfiguration_NamedRabbitMq_PreservesRawUri() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:MessageBus"] = "provider=RaBbItMq", + ["ConnectionStrings:RabbitMQ"] = "'amqp://rabbit.example.test/team%2Fprod'" + }); + + Assert.Equal("amqp://rabbit.example.test/team%2Fprod", options.MessageBusOptions.ConnectionString); + } + + [Fact] + public void ReadFromConfiguration_InlineOpaqueRedis_PreservesRawConnectionString() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Cache"] = "provider=redis;redis:6379,password=p%40ss,abortConnect=false" + }); + + Assert.Equal("redis:6379,password=p%40ss,abortConnect=false", options.CacheOptions.ConnectionString); + } + + [Theory] + [InlineData("ssl=true,redis:6380")] + [InlineData("password=secret,redis:6379")] + [InlineData("serviceName=primary,redis:26379")] + public void ReadFromConfiguration_OptionFirstRedis_PreservesNativeConnectionString(string connectionString) + { + AppOptions options = ReadOptions(new() { ["ConnectionStrings:Redis"] = connectionString }); + + Assert.Equal(connectionString, options.CacheOptions.ConnectionString); + Assert.Equal(connectionString, options.MessageBusOptions.ConnectionString); + Assert.Equal(connectionString, options.QueueOptions.ConnectionString); + Assert.Equal(connectionString, options.CacheOptions.Data["server"]); + Assert.Single(ConfigurationOptions.Parse(connectionString).EndPoints); + } + + [Theory] + [InlineData("provider=redis;ssl=true,redis:6380", "ssl=true,redis:6380")] + [InlineData("provider=redis;server=redis:6379,abortConnect=false", "redis:6379,abortConnect=false")] + public void ReadFromConfiguration_LegacyRedisFullOverride_ProducesNativeConnectionString( + string selector, + string expectedConnectionString) + { + AppOptions options = ReadOptions(new() { ["ConnectionStrings:Cache"] = selector }); + + Assert.Equal(expectedConnectionString, options.CacheOptions.ConnectionString); + Assert.Single(ConfigurationOptions.Parse(expectedConnectionString).EndPoints); + } + + [Fact] + public void ReadFromConfiguration_LegacyRedisPartialOverlay_Throws() + { + var values = new Dictionary + { + ["ConnectionStrings:Redis"] = "redis:6379,abortConnect=false", + ["ConnectionStrings:Cache"] = "provider=redis;server=cache:6380;ssl=true" + }; + + Assert.Throws(() => ReadOptions(values)); + } + + [Fact] + public void ReadFromConfiguration_StructuredTechnologyProviderMetadata_IsRemoved() + { + AppOptions options = ReadOptions(new() + { + ["ConnectionStrings:Storage"] = "provider=s3", + ["ConnectionStrings:S3"] = "provider=s3;bucket=events;region=us-east-2" + }); + + Assert.Equal("bucket=events;region=us-east-2", options.StorageOptions.ConnectionString); + Assert.DoesNotContain("provider", options.StorageOptions.ConnectionString, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ReadFromConfiguration_FolderSelectorWithoutPath_PreservesLegacyDefault() + { + AppOptions options = ReadOptions(new() { ["ConnectionStrings:Storage"] = "provider=folder" }); + + Assert.Equal("folder", options.StorageOptions.Provider); + Assert.Null(options.StorageOptions.ConnectionString); + } + + private static AppOptions ReadOptions(Dictionary values) => ReadOptionsFromConfiguration(CreateConfiguration(values)); + + private static AppOptions ReadOptionsFromConfiguration(IConfiguration configuration) + { + var values = new Dictionary { ["BaseURL"] = "http://localhost" }; + IConfiguration combined = new ConfigurationBuilder() + .AddInMemoryCollection(values) + .AddConfiguration(configuration) + .Build(); + return AppOptions.ReadFromConfiguration(combined); + } + + private static IConfiguration CreateConfiguration(Dictionary values) + { + return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + } +} diff --git a/tests/Exceptionless.Tests/Configuration/RedisConnectionRegistryTests.cs b/tests/Exceptionless.Tests/Configuration/RedisConnectionRegistryTests.cs new file mode 100644 index 0000000000..c9c59937cd --- /dev/null +++ b/tests/Exceptionless.Tests/Configuration/RedisConnectionRegistryTests.cs @@ -0,0 +1,265 @@ +using System.Net; +using System.Reflection; +using Exceptionless.Core; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Insulation.Redis; +using Foundatio.Caching; +using Foundatio.Messaging; +using Foundatio.Queues; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using Xunit; + +namespace Exceptionless.Tests.Configuration; + +public class RedisConnectionRegistryTests +{ + [Fact] + public void GetConnection_EqualStrings_CreatesOneConnection() + { + int creationCount = 0; + using var registry = CreateRegistry(_ => + { + creationCount++; + return CreateConnection(); + }); + + IConnectionMultiplexer first = registry.GetConnection("redis:6379,password=secret"); + IConnectionMultiplexer second = registry.GetConnection("redis:6379,password=secret"); + + Assert.Same(first, second); + Assert.Equal(1, creationCount); + } + + [Fact] + public void GetConnection_DifferentRoleStrings_RemainIsolated() + { + using var registry = CreateRegistry(_ => CreateConnection()); + + IConnectionMultiplexer cache = registry.GetConnection("cache:6379"); + IConnectionMultiplexer messageBus = registry.GetConnection("message-bus:6379"); + IConnectionMultiplexer queue = registry.GetConnection("queue:6379"); + + Assert.NotSame(cache, messageBus); + Assert.NotSame(messageBus, queue); + Assert.NotSame(cache, queue); + } + + [Fact] + public void Dispose_RegistryOwnedConnections_DisposesEachConnectionOnce() + { + var proxies = new List(); + var registry = CreateRegistry(_ => CreateConnection(proxies)); + registry.GetConnection("message-bus:6379"); + registry.GetConnection("message-bus:6379"); + registry.GetConnection("queue:6379"); + + registry.Dispose(); + registry.Dispose(); + + Assert.Equal(2, proxies.Count); + Assert.All(proxies, proxy => Assert.Equal(1, proxy.DisposeCount)); + } + + [Fact] + public void Dispose_CacheCompatibilityConnection_IsDisposedOnceByServiceProviderOwner() + { + var proxies = new List(); + var registry = CreateRegistry(_ => CreateConnection(proxies)); + IConnectionMultiplexer cache = registry.GetCacheConnection("cache:6379"); + + cache.Dispose(); + registry.Dispose(); + + Assert.Single(proxies); + Assert.Equal(1, proxies[0].DisposeCount); + } + + [Fact] + public void RegisterServices_RedisQueueWithLocalCache_RegistersQueueWithoutCacheMultiplexer() + { + AppOptions options = CreateOptions(new() + { + ["ConnectionStrings:Cache"] = "local", + ["ConnectionStrings:MessageBus"] = "local", + ["ConnectionStrings:Queue"] = "provider=redis;queue:6379", + ["ConnectionStrings:Storage"] = "local" + }); + var services = new ServiceCollection(); + services.AddSingleton(NullLoggerFactory.Instance); + + Exceptionless.Insulation.Bootstrapper.RegisterServices(services, options, runMaintenanceTasks: false); + + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(RedisConnectionRegistry)); + Assert.DoesNotContain(services, descriptor => descriptor.ServiceType == typeof(IConnectionMultiplexer)); + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(IQueue)); + } + + [Theory] + [InlineData("server=redis:6379")] + [InlineData("redis:6379;abortConnect=false")] + [InlineData("ssl=true")] + [InlineData("redis:6379;password=redis-password-secret-canary")] + public void RegisterServices_InvalidRedisConnectionString_FailsBeforeServiceResolution(string connectionString) + { + AppOptions options = CreateOptions(new() + { + ["ConnectionStrings:Redis"] = connectionString, + ["ConnectionStrings:Storage"] = "local" + }); + var services = new ServiceCollection(); + + InvalidOperationException exception = Assert.Throws(() => + Exceptionless.Insulation.Bootstrapper.RegisterServices(services, options, runMaintenanceTasks: false)); + + Assert.Contains("Redis connection string selected for Cache is invalid", exception.Message, StringComparison.Ordinal); + Assert.DoesNotContain(connectionString, exception.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("redis-password-secret-canary", exception.ToString(), StringComparison.Ordinal); + } + + [Theory] + [InlineData("redis:6379,abortConnect=false")] + [InlineData("ssl=true,redis:6380")] + [InlineData("password=secret,redis:6379")] + [InlineData("serviceName=primary,redis:26379")] + public void RegisterServices_ValidNativeRedisConnectionString_PassesStartupValidation(string connectionString) + { + AppOptions options = CreateOptions(new() + { + ["ConnectionStrings:Redis"] = connectionString, + ["ConnectionStrings:Storage"] = "local" + }); + var services = new ServiceCollection(); + + Exceptionless.Insulation.Bootstrapper.RegisterServices(services, options, runMaintenanceTasks: false); + + Assert.Contains(services, descriptor => descriptor.ServiceType == typeof(RedisConnectionRegistry)); + } + + [Fact] + public void RegisterServices_DifferentRedisRoleStrings_RequestIsolatedEndpoints() + { + AppOptions options = CreateOptions(new() + { + ["ConnectionStrings:Cache"] = "provider=redis;cache:6379", + ["ConnectionStrings:MessageBus"] = "provider=redis;message-bus:6379", + ["ConnectionStrings:Queue"] = "provider=redis;queue:6379", + ["ConnectionStrings:Storage"] = "local" + }); + var requestedEndpoints = new List(); + var registry = CreateRegistry(endpoint => + { + requestedEndpoints.Add(endpoint); + return CreateConnection(configuration: endpoint); + }); + var services = new ServiceCollection(); + services.AddSingleton(NullLoggerFactory.Instance); + Exceptionless.Core.Bootstrapper.RegisterServices(services, options); + Exceptionless.Insulation.Bootstrapper.RegisterServices(services, options, runMaintenanceTasks: false); + services.RemoveAll(); + services.AddSingleton(registry); + + using ServiceProvider serviceProvider = services.BuildServiceProvider(); + serviceProvider.GetRequiredService(); + serviceProvider.GetRequiredService(); + serviceProvider.GetRequiredService>(); + + Assert.Equal(["cache:6379", "message-bus:6379", "queue:6379"], requestedEndpoints); + } + + private static RedisConnectionRegistry CreateRegistry(Func factory) + { + return new RedisConnectionRegistry( + NullLoggerFactory.Instance, + (connectionString, _) => factory(connectionString)); + } + + private static IConnectionMultiplexer CreateConnection( + List? proxies = null, + string configuration = "localhost:6379") + { + IConnectionMultiplexer connection = DispatchProxy.Create(); + var proxy = (MultiplexerProxy)(object)connection; + proxy.Connection = connection; + proxy.Configuration = configuration; + proxies?.Add(proxy); + return connection; + } + + private static AppOptions CreateOptions(Dictionary values) + { + values["BaseURL"] = "http://localhost"; + IConfiguration configuration = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + return AppOptions.ReadFromConfiguration(configuration); + } + + public class MultiplexerProxy : DispatchProxy + { + public IConnectionMultiplexer Connection { get; set; } = null!; + public string Configuration { get; set; } = null!; + public int DisposeCount { get; private set; } + + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + if (targetMethod?.Name == nameof(IDisposable.Dispose)) + DisposeCount++; + + if (targetMethod?.Name == nameof(IConnectionMultiplexer.GetSubscriber)) + { + ISubscriber subscriber = DispatchProxy.Create(); + ((SubscriberProxy)(object)subscriber).Connection = Connection; + return subscriber; + } + + if (targetMethod?.Name == nameof(IConnectionMultiplexer.GetDatabase)) + return DispatchProxy.Create(); + + if (targetMethod?.Name == nameof(IConnectionMultiplexer.GetEndPoints)) + return Array.Empty(); + + if (targetMethod?.Name == "get_Configuration") + return Configuration; + + return targetMethod?.ReturnType == typeof(void) + ? null + : targetMethod?.ReturnType.IsValueType == true + ? Activator.CreateInstance(targetMethod.ReturnType) + : null; + } + } + + public class SubscriberProxy : DefaultProxy + { + public IConnectionMultiplexer Connection { get; set; } = null!; + + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + if (targetMethod?.Name == "get_Multiplexer") + return Connection; + + return base.Invoke(targetMethod, args); + } + } + + public class DefaultProxy : DispatchProxy + { + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + if (targetMethod?.ReturnType == typeof(void)) + return null; + + if (targetMethod?.ReturnType == typeof(Task)) + return Task.CompletedTask; + + if (targetMethod?.ReturnType.IsValueType == true) + return Activator.CreateInstance(targetMethod.ReturnType); + + return null; + } + } +} diff --git a/tests/Exceptionless.Tests/appsettings.yml b/tests/Exceptionless.Tests/appsettings.yml index 5914619e15..bc29133a60 100644 --- a/tests/Exceptionless.Tests/appsettings.yml +++ b/tests/Exceptionless.Tests/appsettings.yml @@ -2,10 +2,7 @@ ConnectionStrings: # Redis: localhost,abortConnect=false # Elasticsearch: server=https://elastic:elastic@localhost:9200 - # Cache: provider=redis; - # MessageBus: provider=redis; - # Queue: provider=redis; - Storage: provider=folder;path=..\..\..\..\..\src\Exceptionless.Web\storage + Folder: path=..\..\..\..\..\src\Exceptionless.Web\storage # Base url for the ui used to build links in emails and other places. BaseURL: "http://localhost:7110"