From 4736fdee1ad88090e8c1db0d115e40b230048180 Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Mon, 2 Nov 2026 21:04:44 +0100 Subject: [PATCH 1/9] Add opt-in normalization options to content naming --- docs/content-naming-provider.md | 70 ++++++++++++ docs/content-naming.md | 1 + .../ContentNamingProvider.cs | 37 ++++++ src/Storage/Configuration/ContentNaming.cs | 14 +++ .../ContentNamingNormalizationTests.cs | 105 ++++++++++++++++++ 5 files changed, 227 insertions(+) create mode 100644 src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index e3f957792..e1a50dd9e 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -33,6 +33,76 @@ We offer the following predefined naming providers. You may choose either one of Duplicated 'ReplacementChar' characters are replaced by a single character (so after the conversion the name may contain less characters than the display name). This conversion is very simple but there is a chance of non-unique name creation. The original file extension will be kept. +### Optional name normalization + +Two independent settings in `sensenet:contentNaming` control normalization by +`CharReplacementContentNamingProvider`. Both default to `false`, so existing +deployments keep their current naming behavior. + +| Setting | Default | Effect | +| --- | --- | --- | +| `RemoveDiacritics` | `false` | Removes decomposable Unicode diacritics before invalid-character replacement. | +| `ReplaceWhitespace` | `false` | Replaces Unicode whitespace with the configured `ReplacementChar` (default: `-`). | + +For example, with the default replacement character: + +| RemoveDiacritics | ReplaceWhitespace | Display name | Generated name | +| --- | --- | --- | --- | +| false | false | Árvíz tükör | Árvíz tükör | +| true | false | Árvíz tükör | Arviz tukor | +| false | true | Árvíz tükör | Árvíz-tükör | +| true | true | Árvíz tükör | Arviz-tukor | + +Configure an ASP.NET Core repository host in `appsettings.json`: + +```json +{ + "sensenet": { + "contentNaming": { + "RemoveDiacritics": true, + "ReplaceWhitespace": true, + "ReplacementChar": "-" + } + } +} +``` + +Or provide the equivalent environment variables, for example in Docker Compose: + +```yaml +environment: + sensenet__contentNaming__RemoveDiacritics: "true" + sensenet__contentNaming__ReplaceWhitespace: "true" + sensenet__contentNaming__ReplacementChar: "-" +``` + +Restart the repository host after changing these startup settings. The Docker +image must contain a sensenet version that implements these options; adding +environment variables to an older image does not add the feature. No custom +provider registration or content type change is required. + +Normalization preserves letter case and runs on both the original name and the +display name before the existing extension comparison. The original extension +is therefore kept in its normalized form: original `régi.dóc` and display name +`Új név.dóc` produce `Uj-nev.doc`, without duplicating the extension. Existing +replacement-character collapsing and trailing-character trimming still apply; +this feature does not add leading-character trimming or lowercase conversion. + +Diacritic removal uses canonical Unicode decomposition and removes combining +marks, including supplementary Unicode marks. It is not full ASCII +transliteration: letters such as `ø`, `ß`, and non-Latin letters remain. +Combining marks are removed in all scripts, so enable this only where that +naming policy is appropriate. Names can collide after normalization +(`Árvíz` and `Arviz`); the existing duplicate-name and incremental-naming rules +still apply. Whitespace-only or combining-mark-only input can become empty. + +These settings only affect names generated by +`CharReplacementContentNamingProvider`. They do not change `DisplayName`, +rename existing content, change name validation, or affect +`Underscore5FContentNamingProvider`. Clients with local name-generation logic +should use the existing `GetNameFromDisplayName` OData function for matching +previews. + ## Underscore5FContentNamingProvider Contains a customized *GenerateNameFromDisplayName* method that encodes the display name. The encoding works with the standard UrlEncode .Net method, but the percent sign ('%') will be replaced with an underscore ('_') character. The names that are generated from unique names are guaranteed to remain unique. After the conversion, the name may contain more characters than the input name. The original file extension will be kept. Invalid characters are configurable in the **sensenet/contentNaming** section in the web.config. diff --git a/docs/content-naming.md b/docs/content-naming.md index a31874606..cec199550 100644 --- a/docs/content-naming.md +++ b/docs/content-naming.md @@ -122,6 +122,7 @@ Pattern must start with '[' and end with ']'. --> ``` - **InvalidNameCharsPattern**: a regular expression that defines the invalid characters a Content Name may not contain. These invalid characters are automatically encoded (or replaced) during converting a display name to a name, depending on the configured [ContentNamingProvider](content-naming-provider.md). If the Name is not autogenerated (for example for Files when DisplayName Field Control is not visible in the Content View) and the user inputs a name that contains invalid characters, an error message will be displayed upon trying to save the Content. The regular expression may contain characterset with a negating clause ([^...]) thus defining allowed characters instead of invalid ones, or define invalid characters simply ([...]). In both cases the regular expression MUST start with '[' and end with ']'. - **ReplacementChar**: defines the character used to replace invalid characters. +- **RemoveDiacritics** and **ReplaceWhitespace**: optional, independent normalization settings for `CharReplacementContentNamingProvider`, both `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization) for appsettings.json and Docker environment variable examples. > Note that changing InvalidNameCharsPattern will affect path validation logic in the whole Content Repository. It is therefore desired to change validation messages when changing invalid characters pattern, see [Invalid names and error messages](#Invalid-names-and-error-messages) below. diff --git a/src/ContentRepository/ContentNamingProvider.cs b/src/ContentRepository/ContentNamingProvider.cs index b53c17196..2759296b4 100644 --- a/src/ContentRepository/ContentNamingProvider.cs +++ b/src/ContentRepository/ContentNamingProvider.cs @@ -4,6 +4,7 @@ using SenseNet.ContentRepository.Storage.Data; using SenseNet.Diagnostics; using System; +using System.Globalization; using System.Collections.Generic; using System.IO; using System.Linq; @@ -364,6 +365,14 @@ public override string GenerateNameFromDisplayName(string originalName, string d if (string.IsNullOrEmpty(displayName)) return displayName; + var removeDiacritics = ContentNaming.RemoveDiacritics; + var replaceWhitespace = ContentNaming.ReplaceWhitespace; + if (removeDiacritics || replaceWhitespace) + { + originalName = NormalizeName(originalName, removeDiacritics, replaceWhitespace); + displayName = NormalizeName(displayName, removeDiacritics, replaceWhitespace); + } + // note: original extension is kept! // get original extension @@ -387,6 +396,34 @@ public override string GenerateNameFromDisplayName(string originalName, string d return newName.TrimEnd('.', replacementChar); } + private static string NormalizeName(string name, bool removeDiacritics, bool replaceWhitespace) + { + if (string.IsNullOrEmpty(name)) + return name; + + var normalizedName = removeDiacritics ? name.Normalize(NormalizationForm.FormD) : name; + var result = new StringBuilder(normalizedName.Length); + foreach (var rune in normalizedName.EnumerateRunes()) + { + if (removeDiacritics) + { + var category = Rune.GetUnicodeCategory(rune); + if (category == UnicodeCategory.NonSpacingMark || + category == UnicodeCategory.SpacingCombiningMark || + category == UnicodeCategory.EnclosingMark) + continue; + } + + if (replaceWhitespace && Rune.IsWhiteSpace(rune)) + result.Append(replacementChar); + else + result.Append(rune.ToString()); + } + + var normalized = result.ToString(); + return removeDiacritics ? normalized.Normalize(NormalizationForm.FormC) : normalized; + } + private string RemoveInvalidCharacters(string name) { name = Regex.Replace(name, ContentNaming.InvalidNameCharsPattern, replacementString); diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index 14c650760..94c9e9b79 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -18,5 +18,19 @@ public class ContentNaming : SnConfig public static char ReplacementChar { get; internal set; } = GetString(SECTIONNAME, "ReplacementChar", "-").FirstOrDefault(); + + /// + /// Removes decomposable Unicode diacritics when generating a name from a display name. + /// Default: false. + /// + public static bool RemoveDiacritics { get; internal set; } = + GetValue(SECTIONNAME, nameof(RemoveDiacritics), false); + + /// + /// Replaces Unicode whitespace with ReplacementChar when generating a name from a display name. + /// Default: false. + /// + public static bool ReplaceWhitespace { get; internal set; } = + GetValue(SECTIONNAME, nameof(ReplaceWhitespace), false); } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs new file mode 100644 index 000000000..d3bef40d1 --- /dev/null +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs @@ -0,0 +1,105 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SenseNet.Configuration; +using SenseNet.Testing; + +namespace SenseNet.ContentRepository.Tests +{ + [TestClass] + [DoNotParallelize] + public class ContentNamingNormalizationTests + { + [TestMethod] + public void Normalization_IsDisabledByDefault() + { + Assert.IsFalse(ContentNaming.RemoveDiacritics); + Assert.IsFalse(ContentNaming.ReplaceWhitespace); + } + + [DataTestMethod] + [DataRow(false, false, "Árvíz tükör")] + [DataRow(true, false, "Arviz tukor")] + [DataRow(false, true, "Árvíz-tükör")] + [DataRow(true, true, "Arviz-tukor")] + public void Normalization_OptionsAreIndependent(bool removeDiacritics, bool replaceWhitespace, string expected) + { + using var diacritics = new Swindler(removeDiacritics, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(replaceWhitespace, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + + [DataTestMethod] + [DataRow(null, "Árvíztűrő tükörfúrógép", "Arvizturo-tukorfurogep")] + [DataRow(null, "áéíóöőúüű ÁÉÍÓÖŐÚÜŰ", "aeiooouuu-AEIOOOUUU")] + [DataRow(null, "A\u0301rvi\u0301ztu\u030Bro\u030B tu\u0308ko\u0308rfu\u0301ro\u0301ge\u0301p", "Arvizturo-tukorfurogep")] + [DataRow(null, "Árvíz\t\u00a0\u2003tükör\r\n", "Arviz-tukor")] + [DataRow(null, "Árvíz / tükör?", "Arviz-tukor")] + [DataRow(null, "Árvíz-- tükör", "Arviz-tukor")] + [DataRow(null, "Árvíz.txt", "Arviz.txt")] + [DataRow("régi.txt", "Új név", "Uj-nev.txt")] + [DataRow("régi.txt", "Új név.txt", "Uj-nev.txt")] + [DataRow("régi.txt", "Új név.doc", "Uj-nev.doc.txt")] + [DataRow("régi.dóc", "Új név.dóc", "Uj-nev.doc")] + [DataRow(null, "東京 café", "東京-cafe")] + [DataRow(null, "Straße smør", "Straße-smør")] + [DataRow(null, "😀 Árvíz", "😀-Arviz")] + [DataRow(null, "A\U0001d185 B", "A-B")] + [DataRow(null, " Árvíz ", "-Arviz")] + [DataRow(null, "\t\u00a0\r\n", "")] + [DataRow(null, "\u0301\u0308", "")] + [DataRow(null, "", "")] + [DataRow("régi.txt", null, null)] + public void Normalization_HandlesUnicodeAndExtensions(string originalName, string displayName, string expected) + { + using var diacritics = new Swindler(true, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(true, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(originalName, displayName)); + } + + [TestMethod] + public void Normalization_RunsBeforeInvalidCharacterReplacement() + { + using var diacritics = new Swindler(true, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(true, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + using var pattern = new Swindler("[^a-zA-Z0-9.\\-_]", + () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); + + Assert.AreEqual("Arviz-tukor", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + + [TestMethod] + public void Normalization_WhitespaceOnlyKeepsDiacriticsAndTheirUnicodeForm() + { + using var diacritics = new Swindler(false, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(true, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + Assert.AreEqual("A\u0301rvi\u0301z-tu\u0308ko\u0308r", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, + "A\u0301rvi\u0301z tu\u0308ko\u0308r")); + } + + [TestMethod] + public void Normalization_DoesNotChangeTheEncodingProvider() + { + using var diacritics = new Swindler(true, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(true, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + Assert.AreEqual("Árvíz tükör", + new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + } +} From d356db94ccaf8f2d1f86f771717e558144324e1f Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Mon, 2 Nov 2026 21:23:09 +0100 Subject: [PATCH 2/9] Preserve naming semantics and reuse whitespace replacement --- docs/content-naming-provider.md | 18 +++-- .../ContentNamingProvider.cs | 50 ++++++-------- .../ContentNamingNormalizationTests.cs | 28 +++++++- .../ODataContentNamingTests.cs | 68 +++++++++++++++++++ 4 files changed, 126 insertions(+), 38 deletions(-) create mode 100644 src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index e1a50dd9e..9547dde63 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -81,12 +81,18 @@ image must contain a sensenet version that implements these options; adding environment variables to an older image does not add the feature. No custom provider registration or content type change is required. -Normalization preserves letter case and runs on both the original name and the -display name before the existing extension comparison. The original extension -is therefore kept in its normalized form: original `régi.dóc` and display name -`Új név.dóc` produce `Uj-nev.doc`, without duplicating the extension. Existing -replacement-character collapsing and trailing-character trimming still apply; -this feature does not add leading-character trimming or lowercase conversion. +Normalization only transforms the candidate used for the returned `Name` value; +the source arguments and the stored `DisplayName` field are not modified. Letter +case and the existing extension comparison are preserved. If an original name is +provided, its extension is appended unchanged: original `régi.dóc` and display +name `Új név.dóc` produce `Uj-nev.dóc`. Without an original name, the complete +candidate is normalized, so `Új név.dóc` produces `Uj-nev.doc`. + +Whitespace can already be replaced by including it in `InvalidNameCharsPattern`. +`ReplaceWhitespace` extends the existing generation regex with whitespace; it +does not change that shared validation pattern or introduce a second replacement +algorithm. Existing replacement-character collapsing and trailing-character +trimming still apply; no leading-character trimming or lowercase conversion is added. Diacritic removal uses canonical Unicode decomposition and removes combining marks, including supplementary Unicode marks. It is not full ASCII diff --git a/src/ContentRepository/ContentNamingProvider.cs b/src/ContentRepository/ContentNamingProvider.cs index 2759296b4..b11481943 100644 --- a/src/ContentRepository/ContentNamingProvider.cs +++ b/src/ContentRepository/ContentNamingProvider.cs @@ -4,8 +4,8 @@ using SenseNet.ContentRepository.Storage.Data; using SenseNet.Diagnostics; using System; -using System.Globalization; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -365,14 +365,6 @@ public override string GenerateNameFromDisplayName(string originalName, string d if (string.IsNullOrEmpty(displayName)) return displayName; - var removeDiacritics = ContentNaming.RemoveDiacritics; - var replaceWhitespace = ContentNaming.ReplaceWhitespace; - if (removeDiacritics || replaceWhitespace) - { - originalName = NormalizeName(originalName, removeDiacritics, replaceWhitespace); - displayName = NormalizeName(displayName, removeDiacritics, replaceWhitespace); - } - // note: original extension is kept! // get original extension @@ -389,6 +381,10 @@ public override string GenerateNameFromDisplayName(string originalName, string d // convert the displayname to name: if extensions are equal, only filename is to be transformed. extension is added at the end var nameToConvert = origext == newext ? displayNameWithoutExtension : displayName; + // Only transform the candidate for Name; keep both source arguments unchanged. + if (ContentNaming.RemoveDiacritics) + nameToConvert = RemoveDiacritics(nameToConvert); + string newName = RemoveInvalidCharacters(nameToConvert); // attach original extension @@ -396,37 +392,29 @@ public override string GenerateNameFromDisplayName(string originalName, string d return newName.TrimEnd('.', replacementChar); } - private static string NormalizeName(string name, bool removeDiacritics, bool replaceWhitespace) + private static string RemoveDiacritics(string name) { - if (string.IsNullOrEmpty(name)) - return name; - - var normalizedName = removeDiacritics ? name.Normalize(NormalizationForm.FormD) : name; - var result = new StringBuilder(normalizedName.Length); - foreach (var rune in normalizedName.EnumerateRunes()) + var result = new StringBuilder(name.Length); + foreach (var rune in name.Normalize(NormalizationForm.FormD).EnumerateRunes()) { - if (removeDiacritics) - { - var category = Rune.GetUnicodeCategory(rune); - if (category == UnicodeCategory.NonSpacingMark || - category == UnicodeCategory.SpacingCombiningMark || - category == UnicodeCategory.EnclosingMark) - continue; - } - - if (replaceWhitespace && Rune.IsWhiteSpace(rune)) - result.Append(replacementChar); - else + var category = Rune.GetUnicodeCategory(rune); + if (category != UnicodeCategory.NonSpacingMark && + category != UnicodeCategory.SpacingCombiningMark && + category != UnicodeCategory.EnclosingMark) result.Append(rune.ToString()); } - var normalized = result.ToString(); - return removeDiacritics ? normalized.Normalize(NormalizationForm.FormC) : normalized; + return result.ToString().Normalize(NormalizationForm.FormC); } private string RemoveInvalidCharacters(string name) { - name = Regex.Replace(name, ContentNaming.InvalidNameCharsPattern, replacementString); + // Keep the configured validation pattern unchanged. This option only + // extends character replacement while generating the Name field value. + var pattern = ContentNaming.ReplaceWhitespace + ? "(?:" + ContentNaming.InvalidNameCharsPattern + @")|\s" + : ContentNaming.InvalidNameCharsPattern; + name = Regex.Replace(name, pattern, replacementString); var doubleReplacement = new string(replacementChar, 2); var length = name.Length; diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs index d3bef40d1..3e6566626 100644 --- a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs @@ -42,7 +42,8 @@ public void Normalization_OptionsAreIndependent(bool removeDiacritics, bool repl [DataRow("régi.txt", "Új név", "Uj-nev.txt")] [DataRow("régi.txt", "Új név.txt", "Uj-nev.txt")] [DataRow("régi.txt", "Új név.doc", "Uj-nev.doc.txt")] - [DataRow("régi.dóc", "Új név.dóc", "Uj-nev.doc")] + [DataRow("régi.dóc", "Új név.dóc", "Uj-nev.dóc")] + [DataRow(null, "Új név.dóc", "Uj-nev.doc")] [DataRow(null, "東京 café", "東京-cafe")] [DataRow(null, "Straße smør", "Straße-smør")] [DataRow(null, "😀 Árvíz", "😀-Arviz")] @@ -90,6 +91,31 @@ public void Normalization_WhitespaceOnlyKeepsDiacriticsAndTheirUnicodeForm() "A\u0301rvi\u0301z tu\u0308ko\u0308r")); } + [TestMethod] + public void Normalization_ExistingPatternCanAlreadyReplaceWhitespace() + { + using var diacritics = new Swindler(false, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(false, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + using var pattern = new Swindler(@"[\s/]", + () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); + + Assert.AreEqual("Árvíz-tükör", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + + [TestMethod] + public void Normalization_WhitespaceOptionDoesNotChangeNameValidation() + { + using var whitespace = new Swindler(true, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + var provider = new CharReplacementContentNamingProvider(); + provider.AssertNameIsValid("Árvíz tükör"); + Assert.AreEqual("Árvíz-tükör", provider.GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + [TestMethod] public void Normalization_DoesNotChangeTheEncodingProvider() { diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs new file mode 100644 index 000000000..e1da87863 --- /dev/null +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs @@ -0,0 +1,68 @@ +using Task = System.Threading.Tasks.Task; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SenseNet.Configuration; +using SenseNet.ContentRepository; +using SenseNet.Testing; + +namespace SenseNet.ODataTests +{ + [TestClass] + [DoNotParallelize] + public class ODataContentNamingTests : ODataTestBase + { + [DataTestMethod] + [DataRow(false, false, null, "Árvíz tükör")] + [DataRow(true, false, null, "Arviz tukor")] + [DataRow(false, true, null, "Árvíz-tükör")] + [DataRow(true, true, null, "Arviz-tukor")] + [DataRow(false, false, "Kézi név", "Kézi név")] + [DataRow(true, false, "Kézi név", "Kezi nev")] + [DataRow(false, true, "Kézi név", "Kézi-név")] + [DataRow(true, true, "Kézi név", "Kezi-nev")] + public async Task OD_POST_NormalizationOnlyChangesName( + bool removeDiacritics, bool replaceWhitespace, string explicitName, string expectedName) + { + await ODataTestAsync(async () => + { + using var diacritics = new Swindler(removeDiacritics, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var whitespace = new Swindler(replaceWhitespace, + () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); + + const string displayName = "Árvíz tükör"; + var root = CreateTestRoot("ODataNamingTestRoot"); + var model = new JObject + { + ["__ContentType"] = "Folder", + ["DisplayName"] = displayName + }; + if (explicitName != null) + model["Name"] = explicitName; + + var response = await ODataPostAsync("/OData.svc" + root.Path, "", + "models=[" + model.ToString(Formatting.None) + "]").ConfigureAwait(false); + + AssertNoError(response); + var entity = GetEntity(response); + var saved = Content.Load(entity.Id); + Assert.AreEqual(expectedName, entity.Name); + Assert.AreEqual(expectedName, saved.Name); + Assert.AreEqual(displayName, saved.DisplayName, + "Generating Name must not normalize the stored DisplayName."); + + // Updating DisplayName later must neither normalize its value nor rename the content. + const string updatedDisplayName = "Új cím — őű"; + var update = new JObject { ["DisplayName"] = updatedDisplayName }; + response = await ODataPatchAsync("/OData.svc" + saved.Path, "", + "models=[" + update.ToString(Formatting.None) + "]").ConfigureAwait(false); + + AssertNoError(response); + saved = Content.Load(entity.Id); + Assert.AreEqual(expectedName, saved.Name); + Assert.AreEqual(updatedDisplayName, saved.DisplayName); + }).ConfigureAwait(false); + } + } +} From a7fbaee34e407662dbc2929f609316616124b567 Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Mon, 2 Nov 2026 21:36:50 +0100 Subject: [PATCH 3/9] Limit content naming option to diacritic removal --- docs/content-naming-provider.md | 78 ++++----------- docs/content-naming.md | 2 +- .../ContentNamingProvider.cs | 8 +- src/Storage/Configuration/ContentNaming.cs | 7 -- .../ContentNamingNormalizationTests.cs | 96 +++++-------------- .../ODataContentNamingTests.cs | 22 ++--- 6 files changed, 47 insertions(+), 166 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 9547dde63..836b8fb9d 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -33,81 +33,35 @@ We offer the following predefined naming providers. You may choose either one of Duplicated 'ReplacementChar' characters are replaced by a single character (so after the conversion the name may contain less characters than the display name). This conversion is very simple but there is a chance of non-unique name creation. The original file extension will be kept. -### Optional name normalization +### Optional diacritic removal -Two independent settings in `sensenet:contentNaming` control normalization by -`CharReplacementContentNamingProvider`. Both default to `false`, so existing -deployments keep their current naming behavior. - -| Setting | Default | Effect | -| --- | --- | --- | -| `RemoveDiacritics` | `false` | Removes decomposable Unicode diacritics before invalid-character replacement. | -| `ReplaceWhitespace` | `false` | Replaces Unicode whitespace with the configured `ReplacementChar` (default: `-`). | - -For example, with the default replacement character: - -| RemoveDiacritics | ReplaceWhitespace | Display name | Generated name | -| --- | --- | --- | --- | -| false | false | Árvíz tükör | Árvíz tükör | -| true | false | Árvíz tükör | Arviz tukor | -| false | true | Árvíz tükör | Árvíz-tükör | -| true | true | Árvíz tükör | Arviz-tukor | - -Configure an ASP.NET Core repository host in `appsettings.json`: +Set `sensenet:contentNaming:RemoveDiacritics` to `true` to remove decomposable +Unicode diacritics from generated `Name` values. The default is `false`. ```json { "sensenet": { "contentNaming": { - "RemoveDiacritics": true, - "ReplaceWhitespace": true, - "ReplacementChar": "-" + "RemoveDiacritics": true } } } ``` -Or provide the equivalent environment variables, for example in Docker Compose: +For Docker, set `sensenet__contentNaming__RemoveDiacritics=true`. Restart the +repository host after changing this setting; its package/image version must +contain this option. -```yaml -environment: - sensenet__contentNaming__RemoveDiacritics: "true" - sensenet__contentNaming__ReplaceWhitespace: "true" - sensenet__contentNaming__ReplacementChar: "-" -``` +For example, `Árvíz tükör` becomes `Arviz tukor`. Whitespace and invalid characters +continue to follow the existing `InvalidNameCharsPattern` and `ReplacementChar` +settings. Letter case, extension handling, and the stored `DisplayName` are +preserved. Existing content is not renamed, and name validation is unchanged. -Restart the repository host after changing these startup settings. The Docker -image must contain a sensenet version that implements these options; adding -environment variables to an older image does not add the feature. No custom -provider registration or content type change is required. - -Normalization only transforms the candidate used for the returned `Name` value; -the source arguments and the stored `DisplayName` field are not modified. Letter -case and the existing extension comparison are preserved. If an original name is -provided, its extension is appended unchanged: original `régi.dóc` and display -name `Új név.dóc` produce `Uj-nev.dóc`. Without an original name, the complete -candidate is normalized, so `Új név.dóc` produces `Uj-nev.doc`. - -Whitespace can already be replaced by including it in `InvalidNameCharsPattern`. -`ReplaceWhitespace` extends the existing generation regex with whitespace; it -does not change that shared validation pattern or introduce a second replacement -algorithm. Existing replacement-character collapsing and trailing-character -trimming still apply; no leading-character trimming or lowercase conversion is added. - -Diacritic removal uses canonical Unicode decomposition and removes combining -marks, including supplementary Unicode marks. It is not full ASCII -transliteration: letters such as `ø`, `ß`, and non-Latin letters remain. -Combining marks are removed in all scripts, so enable this only where that -naming policy is appropriate. Names can collide after normalization -(`Árvíz` and `Arviz`); the existing duplicate-name and incremental-naming rules -still apply. Whitespace-only or combining-mark-only input can become empty. - -These settings only affect names generated by -`CharReplacementContentNamingProvider`. They do not change `DisplayName`, -rename existing content, change name validation, or affect -`Underscore5FContentNamingProvider`. Clients with local name-generation logic -should use the existing `GetNameFromDisplayName` OData function for matching -previews. +Removal uses canonical Unicode decomposition and removes combining marks in all +scripts; it is not full ASCII transliteration (`ø`, `ß`, and non-Latin letters +remain). Different names can produce the same result, and combining-mark-only +input can become empty; existing duplicate-name handling and validation apply. +This option only affects `CharReplacementContentNamingProvider`. ## Underscore5FContentNamingProvider diff --git a/docs/content-naming.md b/docs/content-naming.md index cec199550..b5b186ad0 100644 --- a/docs/content-naming.md +++ b/docs/content-naming.md @@ -122,7 +122,7 @@ Pattern must start with '[' and end with ']'. --> ``` - **InvalidNameCharsPattern**: a regular expression that defines the invalid characters a Content Name may not contain. These invalid characters are automatically encoded (or replaced) during converting a display name to a name, depending on the configured [ContentNamingProvider](content-naming-provider.md). If the Name is not autogenerated (for example for Files when DisplayName Field Control is not visible in the Content View) and the user inputs a name that contains invalid characters, an error message will be displayed upon trying to save the Content. The regular expression may contain characterset with a negating clause ([^...]) thus defining allowed characters instead of invalid ones, or define invalid characters simply ([...]). In both cases the regular expression MUST start with '[' and end with ']'. - **ReplacementChar**: defines the character used to replace invalid characters. -- **RemoveDiacritics** and **ReplaceWhitespace**: optional, independent normalization settings for `CharReplacementContentNamingProvider`, both `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization) for appsettings.json and Docker environment variable examples. +- **RemoveDiacritics**: optional diacritic removal for generated names, `false` by default. See [optional diacritic removal](content-naming-provider.md#optional-diacritic-removal) for configuration and behavior. > Note that changing InvalidNameCharsPattern will affect path validation logic in the whole Content Repository. It is therefore desired to change validation messages when changing invalid characters pattern, see [Invalid names and error messages](#Invalid-names-and-error-messages) below. diff --git a/src/ContentRepository/ContentNamingProvider.cs b/src/ContentRepository/ContentNamingProvider.cs index b11481943..703bb1850 100644 --- a/src/ContentRepository/ContentNamingProvider.cs +++ b/src/ContentRepository/ContentNamingProvider.cs @@ -381,7 +381,6 @@ public override string GenerateNameFromDisplayName(string originalName, string d // convert the displayname to name: if extensions are equal, only filename is to be transformed. extension is added at the end var nameToConvert = origext == newext ? displayNameWithoutExtension : displayName; - // Only transform the candidate for Name; keep both source arguments unchanged. if (ContentNaming.RemoveDiacritics) nameToConvert = RemoveDiacritics(nameToConvert); @@ -409,12 +408,7 @@ private static string RemoveDiacritics(string name) private string RemoveInvalidCharacters(string name) { - // Keep the configured validation pattern unchanged. This option only - // extends character replacement while generating the Name field value. - var pattern = ContentNaming.ReplaceWhitespace - ? "(?:" + ContentNaming.InvalidNameCharsPattern + @")|\s" - : ContentNaming.InvalidNameCharsPattern; - name = Regex.Replace(name, pattern, replacementString); + name = Regex.Replace(name, ContentNaming.InvalidNameCharsPattern, replacementString); var doubleReplacement = new string(replacementChar, 2); var length = name.Length; diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index 94c9e9b79..0d6759b08 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -25,12 +25,5 @@ public class ContentNaming : SnConfig /// public static bool RemoveDiacritics { get; internal set; } = GetValue(SECTIONNAME, nameof(RemoveDiacritics), false); - - /// - /// Replaces Unicode whitespace with ReplacementChar when generating a name from a display name. - /// Default: false. - /// - public static bool ReplaceWhitespace { get; internal set; } = - GetValue(SECTIONNAME, nameof(ReplaceWhitespace), false); } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs index 3e6566626..1e717d339 100644 --- a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs @@ -9,68 +9,56 @@ namespace SenseNet.ContentRepository.Tests public class ContentNamingNormalizationTests { [TestMethod] - public void Normalization_IsDisabledByDefault() + public void DiacriticRemoval_IsDisabledByDefault() { Assert.IsFalse(ContentNaming.RemoveDiacritics); - Assert.IsFalse(ContentNaming.ReplaceWhitespace); } [DataTestMethod] - [DataRow(false, false, "Árvíz tükör")] - [DataRow(true, false, "Arviz tukor")] - [DataRow(false, true, "Árvíz-tükör")] - [DataRow(true, true, "Arviz-tukor")] - public void Normalization_OptionsAreIndependent(bool removeDiacritics, bool replaceWhitespace, string expected) + [DataRow(false, "Árvíz tükör")] + [DataRow(true, "Arviz tukor")] + public void DiacriticRemoval_IsOptional(bool enabled, string expected) { - using var diacritics = new Swindler(removeDiacritics, + using var diacritics = new Swindler(enabled, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(replaceWhitespace, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); Assert.AreEqual(expected, new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); } [DataTestMethod] - [DataRow(null, "Árvíztűrő tükörfúrógép", "Arvizturo-tukorfurogep")] - [DataRow(null, "áéíóöőúüű ÁÉÍÓÖŐÚÜŰ", "aeiooouuu-AEIOOOUUU")] - [DataRow(null, "A\u0301rvi\u0301ztu\u030Bro\u030B tu\u0308ko\u0308rfu\u0301ro\u0301ge\u0301p", "Arvizturo-tukorfurogep")] - [DataRow(null, "Árvíz\t\u00a0\u2003tükör\r\n", "Arviz-tukor")] - [DataRow(null, "Árvíz / tükör?", "Arviz-tukor")] - [DataRow(null, "Árvíz-- tükör", "Arviz-tukor")] + [DataRow(null, "Árvíztűrő tükörfúrógép", "Arvizturo tukorfurogep")] + [DataRow(null, "áéíóöőúüű ÁÉÍÓÖŐÚÜŰ", "aeiooouuu AEIOOOUUU")] + [DataRow(null, "A\u0301rvi\u0301ztu\u030Bro\u030B tu\u0308ko\u0308rfu\u0301ro\u0301ge\u0301p", "Arvizturo tukorfurogep")] [DataRow(null, "Árvíz.txt", "Arviz.txt")] - [DataRow("régi.txt", "Új név", "Uj-nev.txt")] - [DataRow("régi.txt", "Új név.txt", "Uj-nev.txt")] - [DataRow("régi.txt", "Új név.doc", "Uj-nev.doc.txt")] - [DataRow("régi.dóc", "Új név.dóc", "Uj-nev.dóc")] - [DataRow(null, "Új név.dóc", "Uj-nev.doc")] - [DataRow(null, "東京 café", "東京-cafe")] - [DataRow(null, "Straße smør", "Straße-smør")] - [DataRow(null, "😀 Árvíz", "😀-Arviz")] - [DataRow(null, "A\U0001d185 B", "A-B")] - [DataRow(null, " Árvíz ", "-Arviz")] - [DataRow(null, "\t\u00a0\r\n", "")] + [DataRow("régi.txt", "Új név", "Uj nev.txt")] + [DataRow("régi.txt", "Új név.txt", "Uj nev.txt")] + [DataRow("régi.txt", "Új név.doc", "Uj nev.doc.txt")] + [DataRow("régi.dóc", "Új név.dóc", "Uj nev.dóc")] + [DataRow(null, "Új név.dóc", "Uj nev.doc")] + [DataRow(null, "東京 café", "東京 cafe")] + [DataRow(null, "Straße smør", "Straße smør")] + [DataRow(null, "😀 Árvíz", "😀 Arviz")] + [DataRow(null, "A\U0001d185 B", "A B")] + [DataRow(null, " Árvíz ", " Arviz ")] [DataRow(null, "\u0301\u0308", "")] [DataRow(null, "", "")] [DataRow("régi.txt", null, null)] - public void Normalization_HandlesUnicodeAndExtensions(string originalName, string displayName, string expected) + public void DiacriticRemoval_PreservesExistingNamingRules( + string originalName, string displayName, string expected) { using var diacritics = new Swindler(true, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(true, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); Assert.AreEqual(expected, new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(originalName, displayName)); } [TestMethod] - public void Normalization_RunsBeforeInvalidCharacterReplacement() + public void DiacriticRemoval_RunsBeforeInvalidCharacterReplacement() { using var diacritics = new Swindler(true, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(true, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); using var pattern = new Swindler("[^a-zA-Z0-9.\\-_]", () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); @@ -79,50 +67,10 @@ public void Normalization_RunsBeforeInvalidCharacterReplacement() } [TestMethod] - public void Normalization_WhitespaceOnlyKeepsDiacriticsAndTheirUnicodeForm() - { - using var diacritics = new Swindler(false, - () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(true, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); - - Assert.AreEqual("A\u0301rvi\u0301z-tu\u0308ko\u0308r", - new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, - "A\u0301rvi\u0301z tu\u0308ko\u0308r")); - } - - [TestMethod] - public void Normalization_ExistingPatternCanAlreadyReplaceWhitespace() - { - using var diacritics = new Swindler(false, - () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(false, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); - using var pattern = new Swindler(@"[\s/]", - () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); - - Assert.AreEqual("Árvíz-tükör", - new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); - } - - [TestMethod] - public void Normalization_WhitespaceOptionDoesNotChangeNameValidation() - { - using var whitespace = new Swindler(true, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); - - var provider = new CharReplacementContentNamingProvider(); - provider.AssertNameIsValid("Árvíz tükör"); - Assert.AreEqual("Árvíz-tükör", provider.GenerateNameFromDisplayName(null, "Árvíz tükör")); - } - - [TestMethod] - public void Normalization_DoesNotChangeTheEncodingProvider() + public void DiacriticRemoval_DoesNotChangeTheEncodingProvider() { using var diacritics = new Swindler(true, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(true, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); Assert.AreEqual("Árvíz tükör", new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs index e1da87863..01a17afd3 100644 --- a/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs @@ -13,23 +13,17 @@ namespace SenseNet.ODataTests public class ODataContentNamingTests : ODataTestBase { [DataTestMethod] - [DataRow(false, false, null, "Árvíz tükör")] - [DataRow(true, false, null, "Arviz tukor")] - [DataRow(false, true, null, "Árvíz-tükör")] - [DataRow(true, true, null, "Arviz-tukor")] - [DataRow(false, false, "Kézi név", "Kézi név")] - [DataRow(true, false, "Kézi név", "Kezi nev")] - [DataRow(false, true, "Kézi név", "Kézi-név")] - [DataRow(true, true, "Kézi név", "Kezi-nev")] - public async Task OD_POST_NormalizationOnlyChangesName( - bool removeDiacritics, bool replaceWhitespace, string explicitName, string expectedName) + [DataRow(false, null, "Árvíz tükör")] + [DataRow(true, null, "Arviz tukor")] + [DataRow(false, "Kézi név", "Kézi név")] + [DataRow(true, "Kézi név", "Kezi nev")] + public async Task OD_POST_DiacriticRemovalOnlyChangesName( + bool removeDiacritics, string explicitName, string expectedName) { await ODataTestAsync(async () => { using var diacritics = new Swindler(removeDiacritics, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - using var whitespace = new Swindler(replaceWhitespace, - () => ContentNaming.ReplaceWhitespace, value => ContentNaming.ReplaceWhitespace = value); const string displayName = "Árvíz tükör"; var root = CreateTestRoot("ODataNamingTestRoot"); @@ -49,10 +43,8 @@ await ODataTestAsync(async () => var saved = Content.Load(entity.Id); Assert.AreEqual(expectedName, entity.Name); Assert.AreEqual(expectedName, saved.Name); - Assert.AreEqual(displayName, saved.DisplayName, - "Generating Name must not normalize the stored DisplayName."); + Assert.AreEqual(displayName, saved.DisplayName); - // Updating DisplayName later must neither normalize its value nor rename the content. const string updatedDisplayName = "Új cím — őű"; var update = new JObject { ["DisplayName"] = updatedDisplayName }; response = await ODataPatchAsync("/OData.svc" + saved.Path, "", From 21fc32a805a654ba4853014a931ddceface28a3c Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Tue, 3 Nov 2026 08:42:07 +0100 Subject: [PATCH 4/9] Add opt-in lowercase content names --- docs/content-naming-provider.md | 50 ++++++++++------ docs/content-naming.md | 3 +- .../ContentNamingProvider.cs | 3 + src/Storage/Configuration/ContentNaming.cs | 7 +++ .../ContentNamingNormalizationTests.cs | 57 ++++++++++++++++++- .../ODataContentNamingTests.cs | 27 ++++++--- 6 files changed, 119 insertions(+), 28 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 836b8fb9d..406294d22 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -33,35 +33,49 @@ We offer the following predefined naming providers. You may choose either one of Duplicated 'ReplacementChar' characters are replaced by a single character (so after the conversion the name may contain less characters than the display name). This conversion is very simple but there is a chance of non-unique name creation. The original file extension will be kept. -### Optional diacritic removal +### Optional name normalization -Set `sensenet:contentNaming:RemoveDiacritics` to `true` to remove decomposable -Unicode diacritics from generated `Name` values. The default is `false`. +The following independent settings support consistent generated names, for example +when content names form part of public URLs. Both default to `false`: + +- `sensenet:contentNaming:RemoveDiacritics`: removes decomposable Unicode diacritics. +- `sensenet:contentNaming:Lowercase`: converts letters using invariant casing. ```json { "sensenet": { "contentNaming": { - "RemoveDiacritics": true + "RemoveDiacritics": true, + "Lowercase": true } } } ``` -For Docker, set `sensenet__contentNaming__RemoveDiacritics=true`. Restart the -repository host after changing this setting; its package/image version must -contain this option. - -For example, `Árvíz tükör` becomes `Arviz tukor`. Whitespace and invalid characters -continue to follow the existing `InvalidNameCharsPattern` and `ReplacementChar` -settings. Letter case, extension handling, and the stored `DisplayName` are -preserved. Existing content is not renamed, and name validation is unchanged. - -Removal uses canonical Unicode decomposition and removes combining marks in all -scripts; it is not full ASCII transliteration (`ø`, `ß`, and non-Latin letters -remain). Different names can produce the same result, and combining-mark-only -input can become empty; existing duplicate-name handling and validation apply. -This option only affects `CharReplacementContentNamingProvider`. +For Docker, set `sensenet__contentNaming__RemoveDiacritics=true` and/or +`sensenet__contentNaming__Lowercase=true`. Restart the repository host after +changing these settings; its package/image version must contain the options. + +For example, enabling both converts `Árvíz tükör` to `arviz tukor`. Normalization +runs on the selected Name candidate before the existing invalid-character +replacement. Whitespace continues to follow the existing `InvalidNameCharsPattern` +and `ReplacementChar` settings. To generate `arvizturo-tukorfurogep` from +`Árvíztűrő tükörfúrógép`, enable both options and include whitespace in the existing +invalid-character pattern with `ReplacementChar` set to `-`. + +The stored `DisplayName`, original extension handling, and name validation are +unchanged. An extension retained from `originalName` is appended unchanged, even +with `Lowercase` enabled (`NEW.TXT` with original name `old.TXT` becomes `new.TXT`). +Changing naming configuration does not rename existing content; this is the +standard content naming behavior. Neither option changes the encoding provider. + +Diacritic removal uses canonical Unicode decomposition and removes combining +marks in all scripts. It is not full ASCII transliteration: Latin characters +without a canonical decomposition (such as `æ`, `ø`, and `ß`) remain, as do +Chinese, Cyrillic, and other non-Latin letters. Lowercase can still change the +case of letters in those scripts. Different names can produce the same result, +and combining-mark-only input can become empty; existing duplicate-name handling +and validation apply. These options only affect `CharReplacementContentNamingProvider`. ## Underscore5FContentNamingProvider diff --git a/docs/content-naming.md b/docs/content-naming.md index b5b186ad0..6622cb2fe 100644 --- a/docs/content-naming.md +++ b/docs/content-naming.md @@ -122,7 +122,8 @@ Pattern must start with '[' and end with ']'. --> ``` - **InvalidNameCharsPattern**: a regular expression that defines the invalid characters a Content Name may not contain. These invalid characters are automatically encoded (or replaced) during converting a display name to a name, depending on the configured [ContentNamingProvider](content-naming-provider.md). If the Name is not autogenerated (for example for Files when DisplayName Field Control is not visible in the Content View) and the user inputs a name that contains invalid characters, an error message will be displayed upon trying to save the Content. The regular expression may contain characterset with a negating clause ([^...]) thus defining allowed characters instead of invalid ones, or define invalid characters simply ([...]). In both cases the regular expression MUST start with '[' and end with ']'. - **ReplacementChar**: defines the character used to replace invalid characters. -- **RemoveDiacritics**: optional diacritic removal for generated names, `false` by default. See [optional diacritic removal](content-naming-provider.md#optional-diacritic-removal) for configuration and behavior. +- **RemoveDiacritics**: optional diacritic removal for generated names, `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization) for configuration and behavior. +- **Lowercase**: optional invariant lowercasing of generated names, `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization). > Note that changing InvalidNameCharsPattern will affect path validation logic in the whole Content Repository. It is therefore desired to change validation messages when changing invalid characters pattern, see [Invalid names and error messages](#Invalid-names-and-error-messages) below. diff --git a/src/ContentRepository/ContentNamingProvider.cs b/src/ContentRepository/ContentNamingProvider.cs index 703bb1850..a0a44abbb 100644 --- a/src/ContentRepository/ContentNamingProvider.cs +++ b/src/ContentRepository/ContentNamingProvider.cs @@ -384,6 +384,9 @@ public override string GenerateNameFromDisplayName(string originalName, string d if (ContentNaming.RemoveDiacritics) nameToConvert = RemoveDiacritics(nameToConvert); + if (ContentNaming.Lowercase) + nameToConvert = nameToConvert.ToLowerInvariant(); + string newName = RemoveInvalidCharacters(nameToConvert); // attach original extension diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index 0d6759b08..6310bc722 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -25,5 +25,12 @@ public class ContentNaming : SnConfig /// public static bool RemoveDiacritics { get; internal set; } = GetValue(SECTIONNAME, nameof(RemoveDiacritics), false); + + /// + /// Converts generated names to lowercase using invariant casing. + /// The original extension is preserved. Default: false. + /// + public static bool Lowercase { get; internal set; } = + GetValue(SECTIONNAME, nameof(Lowercase), false); } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs index 1e717d339..c413cfb80 100644 --- a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingNormalizationTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Microsoft.VisualStudio.TestTools.UnitTesting; using SenseNet.Configuration; using SenseNet.Testing; @@ -9,9 +10,10 @@ namespace SenseNet.ContentRepository.Tests public class ContentNamingNormalizationTests { [TestMethod] - public void DiacriticRemoval_IsDisabledByDefault() + public void Normalization_IsDisabledByDefault() { Assert.IsFalse(ContentNaming.RemoveDiacritics); + Assert.IsFalse(ContentNaming.Lowercase); } [DataTestMethod] @@ -67,13 +69,64 @@ public void DiacriticRemoval_RunsBeforeInvalidCharacterReplacement() } [TestMethod] - public void DiacriticRemoval_DoesNotChangeTheEncodingProvider() + public void Normalization_DoesNotChangeTheEncodingProvider() { using var diacritics = new Swindler(true, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var lowercase = new Swindler(true, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + Assert.AreEqual("Árvíz tükör", new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); } + + [DataTestMethod] + [DataRow(false, false, "Árvíz tükör")] + [DataRow(true, false, "Arviz tukor")] + [DataRow(false, true, "árvíz tükör")] + [DataRow(true, true, "arviz tukor")] + public void Normalization_OptionsAreIndependent(bool removeDiacritics, bool lowercase, string expected) + { + using var diacritics = new Swindler(removeDiacritics, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var casing = new Swindler(lowercase, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "Árvíz tükör")); + } + + [DataTestMethod] + [DataRow(null, "ÁRVÍZ TÜKÖR", "árvíz tükör")] + [DataRow(null, "NEW.TXT", "new.txt")] + [DataRow("old.TXT", "NEW.TXT", "new.TXT")] + [DataRow("old.TXT", "NEW", "new.TXT")] + [DataRow("old.TXT", "NEW.DOC", "new.doc.TXT")] + [DataRow(null, "東京 CAFÉ 😀", "東京 café 😀")] + [DataRow(null, "", "")] + [DataRow("old.TXT", null, null)] + public void Lowercase_PreservesExistingNamingRules(string originalName, string displayName, string expected) + { + using var casing = new Swindler(true, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(originalName, displayName)); + } + + [TestMethod] + public void Lowercase_UsesInvariantCasingBeforeInvalidCharacterReplacement() + { + using var casing = new Swindler(true, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + using var culture = new Swindler(new CultureInfo("tr-TR"), + () => CultureInfo.CurrentCulture, value => CultureInfo.CurrentCulture = value); + using var pattern = new Swindler("[^a-z]", + () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); + + Assert.AreEqual("file", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "FILE")); + } } } diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs index 01a17afd3..bb7fb6f69 100644 --- a/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingTests.cs @@ -13,19 +13,32 @@ namespace SenseNet.ODataTests public class ODataContentNamingTests : ODataTestBase { [DataTestMethod] - [DataRow(false, null, "Árvíz tükör")] - [DataRow(true, null, "Arviz tukor")] - [DataRow(false, "Kézi név", "Kézi név")] - [DataRow(true, "Kézi név", "Kezi nev")] - public async Task OD_POST_DiacriticRemovalOnlyChangesName( - bool removeDiacritics, string explicitName, string expectedName) + [DataRow(false, false, false, null, "Árvíztűrő tükörfúrógép")] + [DataRow(true, false, false, null, "Arvizturo tukorfurogep")] + [DataRow(false, true, false, null, "árvíztűrő tükörfúrógép")] + [DataRow(true, true, false, null, "arvizturo tukorfurogep")] + [DataRow(false, false, false, "Kézi név", "Kézi név")] + [DataRow(true, false, false, "Kézi név", "Kezi nev")] + [DataRow(false, true, false, "Kézi név", "kézi név")] + [DataRow(true, true, false, "Kézi név", "kezi nev")] + [DataRow(true, true, true, null, "arvizturo-tukorfurogep")] + [DataRow(true, true, true, "Kézi név", "kezi-nev")] + public async Task OD_POST_NormalizationOnlyChangesName( + bool removeDiacritics, bool lowercase, bool replaceWhitespace, string explicitName, string expectedName) { await ODataTestAsync(async () => { using var diacritics = new Swindler(removeDiacritics, () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - const string displayName = "Árvíz tükör"; + using var casing = new Swindler(lowercase, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + using var pattern = new Swindler(replaceWhitespace + ? ContentNaming.InvalidNameCharsPattern.Insert(ContentNaming.InvalidNameCharsPattern.Length - 1, @"\s") + : ContentNaming.InvalidNameCharsPattern, + () => ContentNaming.InvalidNameCharsPattern, value => ContentNaming.InvalidNameCharsPattern = value); + + const string displayName = "Árvíztűrő tükörfúrógép"; var root = CreateTestRoot("ODataNamingTestRoot"); var model = new JObject { From 0e4bb4e24319240e228b4c1984da07af480701b3 Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Tue, 3 Nov 2026 11:46:33 +0100 Subject: [PATCH 5/9] Add custom naming replacements and runtime Settings overrides --- docs/content-naming-provider.md | 130 ++++++++++++---- docs/content-naming.md | 4 + .../ContentNamingProvider.cs | 34 +++- .../SenseNet.ContentRepository.csproj | 7 + .../ThirdPartyNotices/AnyAscii.txt | 17 ++ src/Storage/Configuration/ContentNaming.cs | 37 +++++ .../ContentNamingFeatureTests.cs | 146 ++++++++++++++++++ .../ODataContentNamingFeatureTests.cs | 136 ++++++++++++++++ 8 files changed, 478 insertions(+), 33 deletions(-) create mode 100644 src/ContentRepository/ThirdPartyNotices/AnyAscii.txt create mode 100644 src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs create mode 100644 src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 406294d22..1b223f38b 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -35,47 +35,117 @@ Duplicated 'ReplacementChar' characters are replaced by a single character (so a ### Optional name normalization -The following independent settings support consistent generated names, for example -when content names form part of public URLs. Both default to `false`: - -- `sensenet:contentNaming:RemoveDiacritics`: removes decomposable Unicode diacritics. -- `sensenet:contentNaming:Lowercase`: converts letters using invariant casing. +The following independent `sensenet:contentNaming` switches all default to `false`. +They affect `CharReplacementContentNamingProvider` only. + +| Setting | Effect | +| --- | --- | +| `ApplyCustomReplacements` | Applies the configured literal replacement pairs in array order. | +| `RemoveDiacritics` | Removes decomposable Unicode diacritics using canonical decomposition. | +| `Transliterate` | Uses AnyAscii 0.3.3 to transliterate Unicode text to ASCII. | +| `Lowercase` | Converts letters using invariant casing. | + +The pipeline is custom replacements, diacritic removal, transliteration, lowercase, +then the existing invalid-character replacement. Each step can be enabled separately. +When all four switches are absent or `false`, name generation remains unchanged. +Transliteration itself removes many diacritics. To preserve accents, disable both +`RemoveDiacritics` and `Transliterate` and avoid custom replacements that remove them. +A Settings veto disables its step; other enabled steps can still have overlapping effects. + +For example, this configuration generates `arvizturo-tukorfurogep` from +`Árvíztűrő tükörfúrógép` while preserving the original stored `DisplayName`: ```json { "sensenet": { "contentNaming": { "RemoveDiacritics": true, - "Lowercase": true + "Lowercase": true, + "Transliterate": true, + "ApplyCustomReplacements": true, + "ReplacementSources": [ + "ä", + "&", + " " + ], + "ReplacementTargets": [ + "ae", + "and", + "-" + ] } } } ``` -For Docker, set `sensenet__contentNaming__RemoveDiacritics=true` and/or -`sensenet__contentNaming__Lowercase=true`. Restart the repository host after -changing these settings; its package/image version must contain the options. - -For example, enabling both converts `Árvíz tükör` to `arviz tukor`. Normalization -runs on the selected Name candidate before the existing invalid-character -replacement. Whitespace continues to follow the existing `InvalidNameCharsPattern` -and `ReplacementChar` settings. To generate `arvizturo-tukorfurogep` from -`Árvíztűrő tükörfúrógép`, enable both options and include whitespace in the existing -invalid-character pattern with `ReplacementChar` set to `-`. - -The stored `DisplayName`, original extension handling, and name validation are -unchanged. An extension retained from `originalName` is appended unchanged, even -with `Lowercase` enabled (`NEW.TXT` with original name `old.TXT` becomes `new.TXT`). -Changing naming configuration does not rename existing content; this is the -standard content naming behavior. Neither option changes the encoding provider. - -Diacritic removal uses canonical Unicode decomposition and removes combining -marks in all scripts. It is not full ASCII transliteration: Latin characters -without a canonical decomposition (such as `æ`, `ø`, and `ß`) remain, as do -Chinese, Cyrillic, and other non-Latin letters. Lowercase can still change the -case of letters in those scripts. Different names can produce the same result, -and combining-mark-only input can become empty; existing duplicate-name handling -and validation apply. These options only affect `CharReplacementContentNamingProvider`. +The replacement arrays are paired by index. Entries are case-sensitive literal +strings, not regular expressions, and can contain multiple characters. An empty +target string deletes a match. Sources must be non-empty strings and both arrays +must have the same length; invalid pairs throw when custom replacement is enabled. +Empty arrays perform no replacements. Use string values, not JSON null values. +Pairs run in order and may cascade: `a -> b`, then `b -> c`, turns `ab` into `cc`. +Custom replacement runs first, so `ä -> ae` can override the usual `ä -> a`. + +For Docker, the four switches are environment variables such as +`sensenet__contentNaming__Transliterate=true`. Array elements use consecutive, +zero-based indexes, for example `sensenet__contentNaming__ReplacementSources__0=ä` +and `sensenet__contentNaming__ReplacementTargets__0=ae`. Do not leave gaps: array +loading stops at the first missing index. The host package/image must contain +these features; changing host configuration or replacement arrays requires a restart. + +### Runtime Settings overrides + +Operators can disable any configured step without restarting the repository by +saving the global `/Root/System/Settings/ContentNaming.settings` content. Use the +existing Settings content editing/management APIs and permissions. For example, +this disables only transliteration: + +```json +{ + "Transliterate": false +} +``` + +Use the same four switch names in the settings file. Settings are checked during +name generation on the running repository and use the existing Settings cache +invalidation. The rules are: + +| Host config | Settings value | Effective behavior | +| --- | --- | --- | +| `false` | Any value or absent | Disabled | +| `true` | Absent or `true` | Enabled | +| `true` | `false` | Disabled | + +Settings cannot enable a feature disabled by host configuration. Removing the +override restores the configured behavior. Replacement arrays and the existing +`InvalidNameCharsPattern`/`ReplacementChar` are not overridden by these settings. +This is a global override; the naming API does not receive a content context path. + +### Preserved behavior and Unicode scope + +The stored `DisplayName`, input parameters, original extension handling, and name +validation are unchanged. An extension retained from `originalName` is appended +unchanged, even with normalization enabled (`NEW.TXT` with original name `old.TXT` +becomes `new.TXT` with `Lowercase`). Changing naming configuration or Settings does +not rename existing content; this is the standard content naming behavior. + +The existing `InvalidNameCharsPattern` and `ReplacementChar` still control invalid +character and whitespace replacement. Custom replacements can replace a space +with `-` without changing the global validation pattern. Changing the existing +invalid-character pattern separately still affects repository-wide Name/Path validation. + +`RemoveDiacritics` removes combining marks in all scripts after canonical +decomposition. It leaves `æ`, `ø`, `ß` and non-Latin letters that have no relevant +decomposition. `Transliterate` additionally handles such Latin letters (`Æ -> Ae`, +`Œ -> Oe`, `ß -> ss`) and other writing systems and symbols. AnyAscii works without +language context; its output is not guaranteed to match language-specific spelling. +It can expand symbols, introduce punctuation, or remove unsupported characters. +The existing invalid-character replacement runs afterward. Transliteration does +not lowercase ASCII letters; the separate `Lowercase` option controls that. + +Different input names can produce the same result or an empty name. Existing +duplicate-name handling and validation apply. AnyAscii is distributed under the +ISC license; its notice is included in the package and copied to output/publish. ## Underscore5FContentNamingProvider diff --git a/docs/content-naming.md b/docs/content-naming.md index 6622cb2fe..29ae74ded 100644 --- a/docs/content-naming.md +++ b/docs/content-naming.md @@ -124,6 +124,10 @@ Pattern must start with '[' and end with ']'. --> - **ReplacementChar**: defines the character used to replace invalid characters. - **RemoveDiacritics**: optional diacritic removal for generated names, `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization) for configuration and behavior. - **Lowercase**: optional invariant lowercasing of generated names, `false` by default. See [optional name normalization](content-naming-provider.md#optional-name-normalization). +- **Transliterate**: optional AnyAscii Unicode-to-ASCII transliteration, `false` by default. +- **ApplyCustomReplacements**: optional ordered literal replacements, `false` by default. `ReplacementSources` and `ReplacementTargets` are paired string arrays. + +All four normalization switches can be disabled at runtime through global `ContentNaming.settings`; Settings cannot enable a switch disabled in host config. See [runtime Settings overrides](content-naming-provider.md#runtime-settings-overrides). > Note that changing InvalidNameCharsPattern will affect path validation logic in the whole Content Repository. It is therefore desired to change validation messages when changing invalid characters pattern, see [Invalid names and error messages](#Invalid-names-and-error-messages) below. diff --git a/src/ContentRepository/ContentNamingProvider.cs b/src/ContentRepository/ContentNamingProvider.cs index a0a44abbb..2a885b0fd 100644 --- a/src/ContentRepository/ContentNamingProvider.cs +++ b/src/ContentRepository/ContentNamingProvider.cs @@ -1,4 +1,5 @@ -using SenseNet.ApplicationModel; +using AnyAscii; +using SenseNet.ApplicationModel; using SenseNet.ContentRepository.Schema; using SenseNet.ContentRepository.Storage; using SenseNet.ContentRepository.Storage.Data; @@ -381,10 +382,16 @@ public override string GenerateNameFromDisplayName(string originalName, string d // convert the displayname to name: if extensions are equal, only filename is to be transformed. extension is added at the end var nameToConvert = origext == newext ? displayNameWithoutExtension : displayName; - if (ContentNaming.RemoveDiacritics) + if (IsEnabled(nameof(ContentNaming.ApplyCustomReplacements), ContentNaming.ApplyCustomReplacements)) + nameToConvert = ApplyCustomReplacements(nameToConvert); + + if (IsEnabled(nameof(ContentNaming.RemoveDiacritics), ContentNaming.RemoveDiacritics)) nameToConvert = RemoveDiacritics(nameToConvert); - if (ContentNaming.Lowercase) + if (IsEnabled(nameof(ContentNaming.Transliterate), ContentNaming.Transliterate)) + nameToConvert = nameToConvert.Transliterate(); + + if (IsEnabled(nameof(ContentNaming.Lowercase), ContentNaming.Lowercase)) nameToConvert = nameToConvert.ToLowerInvariant(); string newName = RemoveInvalidCharacters(nameToConvert); @@ -394,6 +401,27 @@ public override string GenerateNameFromDisplayName(string originalName, string d return newName.TrimEnd('.', replacementChar); } + private static bool IsEnabled(string optionName, bool configuredValue) + { + // Repository settings can veto a configured feature, never enable a disabled one. + return configuredValue && (!Repository.Started() || + Settings.GetValue("ContentNaming", optionName, defaultValue: true)); + } + + private static string ApplyCustomReplacements(string name) + { + var sources = ContentNaming.ReplacementSources; + var targets = ContentNaming.ReplacementTargets; + if (sources.Length != targets.Length || sources.Any(string.IsNullOrEmpty) || targets.Any(t => t == null)) + throw new InvalidOperationException("ContentNaming ReplacementSources and ReplacementTargets must have " + + "the same length, with non-empty sources and non-null targets."); + + for (var i = 0; i < sources.Length; i++) + name = name.Replace(sources[i], targets[i], StringComparison.Ordinal); + + return name; + } + private static string RemoveDiacritics(string name) { var result = new StringBuilder(name.Length); diff --git a/src/ContentRepository/SenseNet.ContentRepository.csproj b/src/ContentRepository/SenseNet.ContentRepository.csproj index 71f97a777..83c15e111 100644 --- a/src/ContentRepository/SenseNet.ContentRepository.csproj +++ b/src/ContentRepository/SenseNet.ContentRepository.csproj @@ -36,6 +36,7 @@ + @@ -56,4 +57,10 @@ + + + + \ No newline at end of file diff --git a/src/ContentRepository/ThirdPartyNotices/AnyAscii.txt b/src/ContentRepository/ThirdPartyNotices/AnyAscii.txt new file mode 100644 index 000000000..f7f0dcc94 --- /dev/null +++ b/src/ContentRepository/ThirdPartyNotices/AnyAscii.txt @@ -0,0 +1,17 @@ +AnyAscii 0.3.3 - https://github.com/anyascii/anyascii + +ISC License + +Copyright (c) 2020-2025, Hunter WB + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index 6310bc722..abb042ec1 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Collections.Generic; namespace SenseNet.Configuration { @@ -32,5 +33,41 @@ public class ContentNaming : SnConfig /// public static bool Lowercase { get; internal set; } = GetValue(SECTIONNAME, nameof(Lowercase), false); + + /// + /// Transliterates generated names to ASCII using AnyAscii. Default: false. + /// + public static bool Transliterate { get; internal set; } = + GetValue(SECTIONNAME, nameof(Transliterate), false); + + /// + /// Applies the ordered ReplacementSources/ReplacementTargets pairs to generated names. Default: false. + /// + public static bool ApplyCustomReplacements { get; internal set; } = + GetValue(SECTIONNAME, nameof(ApplyCustomReplacements), false); + + /// + /// Literal, case-sensitive source strings for custom replacements, in application order. + /// + public static string[] ReplacementSources { get; internal set; } = + GetReplacementArray(nameof(ReplacementSources)); + + /// + /// Replacement strings paired by index with ReplacementSources. Empty strings remove matches. + /// + public static string[] ReplacementTargets { get; internal set; } = + GetReplacementArray(nameof(ReplacementTargets)); + + private static string[] GetReplacementArray(string key) + { + var values = new List(); + for (var index = 0; ; index++) + { + var value = GetString(SECTIONNAME, key + ":" + index); + if (value == null) + return values.ToArray(); + values.Add(value); + } + } } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs new file mode 100644 index 000000000..be6ab43da --- /dev/null +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs @@ -0,0 +1,146 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SenseNet.Configuration; +using SenseNet.Testing; + +namespace SenseNet.ContentRepository.Tests +{ + [TestClass] + [DoNotParallelize] + public class ContentNamingFeatureTests + { + [TestMethod] + public void NewOptions_AreDisabledAndMappingsEmptyByDefault() + { + Assert.IsFalse(ContentNaming.Transliterate); + Assert.IsFalse(ContentNaming.ApplyCustomReplacements); + Assert.AreEqual(0, ContentNaming.ReplacementSources.Length); + Assert.AreEqual(0, ContentNaming.ReplacementTargets.Length); + } + + [DataTestMethod] + [DataRow(false, "Ægir Straße smør", "Ægir Straße smør")] + [DataRow(true, "Ægir Straße smør", "Aegir Strasse smor")] + [DataRow(true, "Łódź Œuvre Þing", "Lodz Oeuvre Thing")] + [DataRow(true, "Борис", "Boris")] + [DataRow(true, "深圳", "ShenZhen")] + [DataRow(true, "A\u0301rvi\u0301z", "Arviz")] + [DataRow(true, "👑", "-crown")] + public void Transliteration_IsOptionalAndUsesAnyAscii(bool enabled, string input, string expected) + { + using var option = new Swindler(enabled, + () => ContentNaming.Transliterate, value => ContentNaming.Transliterate = value); + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, input)); + } + + [DataTestMethod] + [DataRow("ä", "ae", "Fähre", "Faehre")] + [DataRow("ab", "xy", "abABab", "xyABxy")] + [DataRow("!", "", "Name!", "Name")] + [DataRow(";", ",", "a;b", "a-b")] + [DataRow(" ", "-", "Árvíztűrő tükörfúrógép", "Árvíztűrő-tükörfúrógép")] + [DataRow("😀", "smile", "😀 Name", "smile Name")] + public void CustomReplacements_AreLiteralCaseSensitiveAndRunBeforeValidation( + string source, string target, string input, string expected) + { + using var option = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var sources = new Swindler(new[] { source }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { target }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, input)); + } + + [DataTestMethod] + [DataRow(false, false, false, false, "ß Á")] + [DataRow(false, false, false, true, "ß á")] + [DataRow(false, true, false, false, "ß A")] + [DataRow(false, true, false, true, "ß a")] + [DataRow(false, false, true, false, "ss A")] + [DataRow(false, false, true, true, "ss a")] + [DataRow(false, true, true, false, "ss A")] + [DataRow(false, true, true, true, "ss a")] + [DataRow(true, false, false, false, "Ä Á")] + [DataRow(true, false, false, true, "ä á")] + [DataRow(true, true, false, false, "A A")] + [DataRow(true, true, false, true, "a a")] + [DataRow(true, false, true, false, "A A")] + [DataRow(true, false, true, true, "a a")] + [DataRow(true, true, true, false, "A A")] + [DataRow(true, true, true, true, "a a")] + public void FourOptions_AreIndependentAndCustomReplacementsRunFirst( + bool replace, bool removeDiacritics, bool transliterate, bool lowercase, string expected) + { + using var replacement = new Swindler(replace, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var diacritics = new Swindler(removeDiacritics, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + using var ascii = new Swindler(transliterate, + () => ContentNaming.Transliterate, value => ContentNaming.Transliterate = value); + using var casing = new Swindler(lowercase, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + using var sources = new Swindler(new[] { "ß" }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { "Ä" }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "ß Á")); + } + + [TestMethod] + public void CustomReplacements_AreAppliedInConfiguredOrder() + { + using var option = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var sources = new Swindler(new[] { "a", "b" }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { "b", "c" }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + + Assert.AreEqual("cc", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, "ab")); + } + + [DataTestMethod] + [DataRow(new[] { "a" }, new string[0])] + [DataRow(new[] { "" }, new[] { "x" })] + [DataRow(new string[] { null }, new[] { "x" })] + [DataRow(new[] { "a" }, new string[] { null })] + public void CustomReplacements_RejectInvalidPairsOnlyWhenEnabled(string[] from, string[] to) + { + using var sources = new Swindler(from, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(to, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + var provider = new CharReplacementContentNamingProvider(); + Assert.AreEqual("Name", provider.GenerateNameFromDisplayName(null, "Name")); + + using var option = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + Assert.ThrowsException(() => provider.GenerateNameFromDisplayName(null, "Name")); + } + + [TestMethod] + public void NewFeatures_PreserveOriginalExtensionAndEncodingProvider() + { + using var replacement = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var ascii = new Swindler(true, + () => ContentNaming.Transliterate, value => ContentNaming.Transliterate = value); + using var sources = new Swindler(new[] { "Æ" }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { "Œ" }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + + Assert.AreEqual("Oegir.DÓC", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName("old.DÓC", "Ægir.DÓC")); + Assert.AreEqual("Ægir.DÓC", + new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Ægir.DÓC")); + } + } +} diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs new file mode 100644 index 000000000..765e953eb --- /dev/null +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs @@ -0,0 +1,136 @@ +using System; +using System.Threading; +using Task = System.Threading.Tasks.Task; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SenseNet.Configuration; +using SenseNet.ContentRepository; +using SenseNet.ContentRepository.Storage; +using SenseNet.Testing; +using Settings = SenseNet.ContentRepository.Settings; + +namespace SenseNet.ODataTests +{ + [TestClass] + [DoNotParallelize] + public class ODataContentNamingFeatureTests : ODataTestBase + { + [DataTestMethod] + [DataRow(true, false, false, "Észak Straße Árvíz")] + [DataRow(false, true, false, "Aegir Strasse Arviz")] + [DataRow(true, true, false, "Eszak Strasse Arviz")] + [DataRow(true, true, true, "eszak strasse arviz")] + public async Task OD_POST_NewFeaturesPreserveDisplayName(bool replace, bool transliterate, bool lowercase, + string expectedName) + { + await ODataTestAsync(async () => + { + using var replacement = ConfigureOption(nameof(ContentNaming.ApplyCustomReplacements), replace); + using var ascii = ConfigureOption(nameof(ContentNaming.Transliterate), transliterate); + using var casing = ConfigureOption(nameof(ContentNaming.Lowercase), lowercase); + using var sources = new Swindler(new[] { "Ægir" }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { "Észak" }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + + var saved = await CreateFolderAsync("Ægir Straße Árvíz", expectedName).ConfigureAwait(false); + const string updatedDisplayName = "Új cím — őű"; + var response = await ODataPatchAsync("/OData.svc" + saved.Path, "", + "models=[" + new JObject { ["DisplayName"] = updatedDisplayName }.ToString(Formatting.None) + "]") + .ConfigureAwait(false); + AssertNoError(response); + saved = Content.Load(saved.Id); + Assert.AreEqual(expectedName, saved.Name); + Assert.AreEqual(updatedDisplayName, saved.DisplayName); + }).ConfigureAwait(false); + } + + [DataTestMethod] + [DataRow(nameof(ContentNaming.RemoveDiacritics), "Straße ARVIZ")] + [DataRow(nameof(ContentNaming.Lowercase), "straße árvíz")] + [DataRow(nameof(ContentNaming.Transliterate), "Strasse ARVIZ")] + [DataRow(nameof(ContentNaming.ApplyCustomReplacements), "Új ÁRVÍZ")] + public async Task OD_Settings_CanVetoEachConfiguredOptionWithoutRestart(string option, string enabledName) + { + await ODataTestAsync(async () => + { + Assert.IsTrue(Repository.Started(), "The Settings override must be tested on a running repository."); + using var feature = ConfigureOption(option, true); + using var sources = new Swindler(new[] { "Straße" }, + () => ContentNaming.ReplacementSources, value => ContentNaming.ReplacementSources = value); + using var targets = new Swindler(new[] { "Új" }, + () => ContentNaming.ReplacementTargets, value => ContentNaming.ReplacementTargets = value); + const string original = "Straße ÁRVÍZ"; + + // No settings file: use the configured feature. + var before = await CreateFolderAsync(original, enabledName).ConfigureAwait(false); + var settings = new Settings(Node.LoadNode(Repository.SettingsFolderPath)) + { + Name = "ContentNaming.settings" + }; + settings.Binary.SetStream(RepositoryTools.GetStreamFromString("{}")); + await settings.SaveAsync(CancellationToken.None).ConfigureAwait(false); + + // No key: still use the configured feature. + await CreateFolderAsync(original, enabledName).ConfigureAwait(false); + + // Persist changes through the existing OData Settings API. Do not clear caches or restart. + await WriteGlobalSettingsAsync(new JObject { [option] = false }).ConfigureAwait(false); + await CreateFolderAsync(original, original).ConfigureAwait(false); + Assert.AreEqual(enabledName, Content.Load(before.Id).Name, "Existing content must retain its name."); + + await WriteGlobalSettingsAsync(new JObject { [option] = true }).ConfigureAwait(false); + await CreateFolderAsync(original, enabledName).ConfigureAwait(false); + + await WriteGlobalSettingsAsync(new JObject()).ConfigureAwait(false); + await CreateFolderAsync(original, enabledName).ConfigureAwait(false); + + // Settings true cannot enable a feature that the host configuration disables. + await WriteGlobalSettingsAsync(new JObject { [option] = true }).ConfigureAwait(false); + using var disabled = ConfigureOption(option, false); + await CreateFolderAsync(original, original).ConfigureAwait(false); + }).ConfigureAwait(false); + } + + private async System.Threading.Tasks.Task CreateFolderAsync(string displayName, string expectedName) + { + var root = CreateTestRoot(); + var model = new JObject { ["__ContentType"] = "Folder", ["DisplayName"] = displayName }; + var response = await ODataPostAsync("/OData.svc" + root.Path, "", + "models=[" + model.ToString(Formatting.None) + "]").ConfigureAwait(false); + AssertNoError(response); + var entity = GetEntity(response); + var saved = Content.Load(entity.Id); + Assert.AreEqual(expectedName, entity.Name); + Assert.AreEqual(expectedName, saved.Name); + Assert.AreEqual(displayName, saved.DisplayName); + return saved; + } + + private async Task WriteGlobalSettingsAsync(JObject values) + { + var model = new JObject { ["name"] = "ContentNaming", ["settingsData"] = values }; + var response = await ODataPostAsync("/OData.svc/Root('System')/WriteSettings", null, + "models=[" + model.ToString(Formatting.None) + "]").ConfigureAwait(false); + AssertNoError(response); + Assert.AreEqual(204, response.StatusCode); + } + + private static Swindler ConfigureOption(string name, bool enabled) + { + return name switch + { + nameof(ContentNaming.RemoveDiacritics) => new Swindler(enabled, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value), + nameof(ContentNaming.Lowercase) => new Swindler(enabled, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value), + nameof(ContentNaming.Transliterate) => new Swindler(enabled, + () => ContentNaming.Transliterate, value => ContentNaming.Transliterate = value), + nameof(ContentNaming.ApplyCustomReplacements) => new Swindler(enabled, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value), + _ => throw new ArgumentOutOfRangeException(nameof(name)) + }; + } + } +} From 7d2d30d2274ff361f2ff4bac9098f3402619074d Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Thu, 5 Nov 2026 11:42:21 +0100 Subject: [PATCH 6/9] Add opt-in European Latin replacement defaults --- docs/content-naming-provider.md | 43 ++++++++++++++++- src/Storage/Configuration/ContentNaming.cs | 30 ++++++++++-- .../ContentNamingFeatureTests.cs | 48 +++++++++++++++++-- .../ODataContentNamingFeatureTests.cs | 13 +++++ 4 files changed, 126 insertions(+), 8 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 1b223f38b..54a798ae3 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -78,11 +78,52 @@ For example, this configuration generates `arvizturo-tukorfurogep` from } ``` +### Prepared European Latin replacements + +When neither indexed replacement array is configured, the following preset is +available. **All four feature switches still default to `false`**: preparing these +pairs does not enable replacement or change existing name generation. + +| Sources | Targets | +| --- | --- | +| `ß`, `ẞ` | `ss`, `SS` | +| `æ`, `Æ`, `œ`, `Œ` | `ae`, `AE`, `oe`, `OE` | +| `ø`, `Ø`, `ł`, `Ł` | `o`, `O`, `l`, `L` | +| `đ`, `Đ`, `ð`, `Ð`, `þ`, `Þ` | `d`, `D`, `d`, `D`, `th`, `TH` | +| `ħ`, `Ħ`, `ı`, `ij`, `IJ` | `h`, `H`, `i`, `ij`, `IJ` | +| `ŋ`, `Ŋ`, `ŧ`, `Ŧ`, `ſ`, `ŀ`, `Ŀ` | `n`, `N`, `t`, `T`, `s`, `l`, `L` | + +Uppercase expansions stay uppercase; enable `Lowercase` for lowercase URLs. +These non-decomposing Latin letter fallbacks follow the +[Unicode CLDR Latin-ASCII conventions](https://github.com/unicode-org/cldr/blob/main/common/transforms/Latin-ASCII.xml). +Ordinary accented letters, including `ä`, `ö` and `ü`, are handled by +`RemoveDiacritics` (`a`, `o`, `u`). Language-specific German `ae`/`oe`/`ue` +spellings can be supplied as custom pairs, as in the earlier `ä -> ae` example; +they are not global defaults because the same letters occur in other languages. + +To use the preset with ordinary accented Latin letters, enable +`ApplyCustomReplacements` and `RemoveDiacritics`; no replacement arrays are needed. +For example, `Fähre Straße Łódź` becomes `Fahre Strasse Lodz`. `Transliterate` +can remain `false`; lowercase is an independent choice. + +These are configurable fallbacks for common Latin-script European text, not a +guarantee for every European language or language-specific spelling. Greek, +Cyrillic and other scripts require the separate `Transliterate` option for ASCII +transliteration. The preset does not replace spaces or punctuation. + +### Configuring replacement pairs + +Supplying both indexed arrays **replaces the complete preset**, rather than +appending to it. To retain a preset pair alongside your own pairs, include it +in your configuration. A missing half of a custom mapping still fails validation +when enabled. Empty JSON arrays supply no indexed pairs, so they select the +preset; use `ApplyCustomReplacements: false` to disable custom replacement. + The replacement arrays are paired by index. Entries are case-sensitive literal strings, not regular expressions, and can contain multiple characters. An empty target string deletes a match. Sources must be non-empty strings and both arrays must have the same length; invalid pairs throw when custom replacement is enabled. -Empty arrays perform no replacements. Use string values, not JSON null values. +Use string values, not JSON null values. Pairs run in order and may cascade: `a -> b`, then `b -> c`, turns `ab` into `cc`. Custom replacement runs first, so `ä -> ae` can override the usual `ä -> a`. diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index abb042ec1..5bed18a25 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -48,24 +48,46 @@ public class ContentNaming : SnConfig /// /// Literal, case-sensitive source strings for custom replacements, in application order. + /// Defaults to common European Latin letters without canonical decomposition. + /// Configured indexed pairs replace the complete preset. /// public static string[] ReplacementSources { get; internal set; } = - GetReplacementArray(nameof(ReplacementSources)); + GetReplacementArray(nameof(ReplacementSources), new[] + { + "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ" + }); /// /// Replacement strings paired by index with ReplacementSources. Empty strings remove matches. /// public static string[] ReplacementTargets { get; internal set; } = - GetReplacementArray(nameof(ReplacementTargets)); + GetReplacementArray(nameof(ReplacementTargets), new[] + { + "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L" + }); - private static string[] GetReplacementArray(string key) + private static string[] GetReplacementArray(string key, string[] defaultValues) { var values = new List(); for (var index = 0; ; index++) { var value = GetString(SECTIONNAME, key + ":" + index); if (value == null) - return values.ToArray(); + { + // Use the preset only when neither indexed array is configured. + // A missing half of a custom pair must still fail validation when enabled. + return values.Count == 0 && + GetString(SECTIONNAME, nameof(ReplacementSources) + ":0") == null && + GetString(SECTIONNAME, nameof(ReplacementTargets) + ":0") == null + ? defaultValues + : values.ToArray(); + } values.Add(value); } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs index be6ab43da..e5aa9fdcd 100644 --- a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs @@ -10,12 +10,54 @@ namespace SenseNet.ContentRepository.Tests public class ContentNamingFeatureTests { [TestMethod] - public void NewOptions_AreDisabledAndMappingsEmptyByDefault() + public void NewOptions_AreDisabledAndEuropeanMappingsPreparedByDefault() { + Assert.IsFalse(ContentNaming.RemoveDiacritics); + Assert.IsFalse(ContentNaming.Lowercase); Assert.IsFalse(ContentNaming.Transliterate); Assert.IsFalse(ContentNaming.ApplyCustomReplacements); - Assert.AreEqual(0, ContentNaming.ReplacementSources.Length); - Assert.AreEqual(0, ContentNaming.ReplacementTargets.Length); + Assert.AreEqual(28, ContentNaming.ReplacementSources.Length); + Assert.AreEqual(ContentNaming.ReplacementSources.Length, ContentNaming.ReplacementTargets.Length); + CollectionAssert.AllItemsAreUnique(ContentNaming.ReplacementSources); + + const string original = "Ärger Straße Ærø Œuvre Łódź Þór Ħobż"; + Assert.AreEqual(original, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, original)); + } + + [DataTestMethod] + [DataRow("Fähre Straße Übergröße", "Fahre Strasse Ubergrosse")] + [DataRow("Ægir Œuvre Øresund", "AEgir OEuvre Oresund")] + [DataRow("Łódź Đakovo", "Lodz Dakovo")] + [DataRow("Þór Ðóra", "THor Dora")] + [DataRow("Ħobż ıslak İstanbul", "Hobz islak Istanbul")] + [DataRow("IJssel Ŋŋ Ŧŧ ſ ŀĿ", "IJssel Nn Tt s lL")] + [DataRow("Őrült tükörfúrógép", "Orult tukorfurogep")] + [DataRow("Ärger Öl Über", "Arger Ol Uber")] + [DataRow("A\u0308rger O\u0308l U\u0308ber", "Arger Ol Uber")] + public void PreparedEuropeanReplacements_ComplementDiacriticRemovalWithoutAnyAscii(string input, string expected) + { + using var replacement = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var diacritics = new Swindler(true, + () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); + + Assert.AreEqual(expected, + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, input)); + } + + [TestMethod] + public void PreparedEuropeanReplacements_KeepOriginalExtensionAndEncodingProvider() + { + using var replacement = new Swindler(true, + () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); + using var casing = new Swindler(true, + () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); + + Assert.AreEqual("strasse.DÓC", + new CharReplacementContentNamingProvider().GenerateNameFromDisplayName("old.DÓC", "Straße.DÓC")); + Assert.AreEqual("Straße.DÓC", + new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Straße.DÓC")); } [DataTestMethod] diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs index 765e953eb..d53b3e117 100644 --- a/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs @@ -46,6 +46,19 @@ await ODataTestAsync(async () => }).ConfigureAwait(false); } + [DataTestMethod] + [DataRow(false, "Ægir Straße Łódź Árvíztűrő tükörfúrógép")] + [DataRow(true, "AEgir Strasse Lodz Arvizturo tukorfurogep")] + public async Task OD_POST_PreparedLatinMappingsPreserveDisplayName(bool enabled, string expectedName) + { + await ODataTestAsync(async () => + { + using var replacement = ConfigureOption(nameof(ContentNaming.ApplyCustomReplacements), enabled); + using var diacritics = ConfigureOption(nameof(ContentNaming.RemoveDiacritics), enabled); + await CreateFolderAsync("Ægir Straße Łódź Árvíztűrő tükörfúrógép", expectedName).ConfigureAwait(false); + }).ConfigureAwait(false); + } + [DataTestMethod] [DataRow(nameof(ContentNaming.RemoveDiacritics), "Straße ARVIZ")] [DataRow(nameof(ContentNaming.Lowercase), "straße árvíz")] From b0ecc7837f24b6ed92e493afcfcf887b50e3866e Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Thu, 5 Nov 2026 11:51:57 +0100 Subject: [PATCH 7/9] Revert "Add opt-in European Latin replacement defaults" This reverts commit 7d2d30d2274ff361f2ff4bac9098f3402619074d. --- docs/content-naming-provider.md | 43 +---------------- src/Storage/Configuration/ContentNaming.cs | 30 ++---------- .../ContentNamingFeatureTests.cs | 48 ++----------------- .../ODataContentNamingFeatureTests.cs | 13 ----- 4 files changed, 8 insertions(+), 126 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 54a798ae3..1b223f38b 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -78,52 +78,11 @@ For example, this configuration generates `arvizturo-tukorfurogep` from } ``` -### Prepared European Latin replacements - -When neither indexed replacement array is configured, the following preset is -available. **All four feature switches still default to `false`**: preparing these -pairs does not enable replacement or change existing name generation. - -| Sources | Targets | -| --- | --- | -| `ß`, `ẞ` | `ss`, `SS` | -| `æ`, `Æ`, `œ`, `Œ` | `ae`, `AE`, `oe`, `OE` | -| `ø`, `Ø`, `ł`, `Ł` | `o`, `O`, `l`, `L` | -| `đ`, `Đ`, `ð`, `Ð`, `þ`, `Þ` | `d`, `D`, `d`, `D`, `th`, `TH` | -| `ħ`, `Ħ`, `ı`, `ij`, `IJ` | `h`, `H`, `i`, `ij`, `IJ` | -| `ŋ`, `Ŋ`, `ŧ`, `Ŧ`, `ſ`, `ŀ`, `Ŀ` | `n`, `N`, `t`, `T`, `s`, `l`, `L` | - -Uppercase expansions stay uppercase; enable `Lowercase` for lowercase URLs. -These non-decomposing Latin letter fallbacks follow the -[Unicode CLDR Latin-ASCII conventions](https://github.com/unicode-org/cldr/blob/main/common/transforms/Latin-ASCII.xml). -Ordinary accented letters, including `ä`, `ö` and `ü`, are handled by -`RemoveDiacritics` (`a`, `o`, `u`). Language-specific German `ae`/`oe`/`ue` -spellings can be supplied as custom pairs, as in the earlier `ä -> ae` example; -they are not global defaults because the same letters occur in other languages. - -To use the preset with ordinary accented Latin letters, enable -`ApplyCustomReplacements` and `RemoveDiacritics`; no replacement arrays are needed. -For example, `Fähre Straße Łódź` becomes `Fahre Strasse Lodz`. `Transliterate` -can remain `false`; lowercase is an independent choice. - -These are configurable fallbacks for common Latin-script European text, not a -guarantee for every European language or language-specific spelling. Greek, -Cyrillic and other scripts require the separate `Transliterate` option for ASCII -transliteration. The preset does not replace spaces or punctuation. - -### Configuring replacement pairs - -Supplying both indexed arrays **replaces the complete preset**, rather than -appending to it. To retain a preset pair alongside your own pairs, include it -in your configuration. A missing half of a custom mapping still fails validation -when enabled. Empty JSON arrays supply no indexed pairs, so they select the -preset; use `ApplyCustomReplacements: false` to disable custom replacement. - The replacement arrays are paired by index. Entries are case-sensitive literal strings, not regular expressions, and can contain multiple characters. An empty target string deletes a match. Sources must be non-empty strings and both arrays must have the same length; invalid pairs throw when custom replacement is enabled. -Use string values, not JSON null values. +Empty arrays perform no replacements. Use string values, not JSON null values. Pairs run in order and may cascade: `a -> b`, then `b -> c`, turns `ab` into `cc`. Custom replacement runs first, so `ä -> ae` can override the usual `ä -> a`. diff --git a/src/Storage/Configuration/ContentNaming.cs b/src/Storage/Configuration/ContentNaming.cs index 5bed18a25..abb042ec1 100644 --- a/src/Storage/Configuration/ContentNaming.cs +++ b/src/Storage/Configuration/ContentNaming.cs @@ -48,46 +48,24 @@ public class ContentNaming : SnConfig /// /// Literal, case-sensitive source strings for custom replacements, in application order. - /// Defaults to common European Latin letters without canonical decomposition. - /// Configured indexed pairs replace the complete preset. /// public static string[] ReplacementSources { get; internal set; } = - GetReplacementArray(nameof(ReplacementSources), new[] - { - "ß", "ẞ", - "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", - "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", - "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ" - }); + GetReplacementArray(nameof(ReplacementSources)); /// /// Replacement strings paired by index with ReplacementSources. Empty strings remove matches. /// public static string[] ReplacementTargets { get; internal set; } = - GetReplacementArray(nameof(ReplacementTargets), new[] - { - "ss", "SS", - "ae", "AE", "oe", "OE", "o", "O", "l", "L", - "d", "D", "d", "D", "th", "TH", "h", "H", - "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L" - }); + GetReplacementArray(nameof(ReplacementTargets)); - private static string[] GetReplacementArray(string key, string[] defaultValues) + private static string[] GetReplacementArray(string key) { var values = new List(); for (var index = 0; ; index++) { var value = GetString(SECTIONNAME, key + ":" + index); if (value == null) - { - // Use the preset only when neither indexed array is configured. - // A missing half of a custom pair must still fail validation when enabled. - return values.Count == 0 && - GetString(SECTIONNAME, nameof(ReplacementSources) + ":0") == null && - GetString(SECTIONNAME, nameof(ReplacementTargets) + ":0") == null - ? defaultValues - : values.ToArray(); - } + return values.ToArray(); values.Add(value); } } diff --git a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs index e5aa9fdcd..be6ab43da 100644 --- a/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs +++ b/src/Tests/SenseNet.ContentRepository.Tests/ContentNamingFeatureTests.cs @@ -10,54 +10,12 @@ namespace SenseNet.ContentRepository.Tests public class ContentNamingFeatureTests { [TestMethod] - public void NewOptions_AreDisabledAndEuropeanMappingsPreparedByDefault() + public void NewOptions_AreDisabledAndMappingsEmptyByDefault() { - Assert.IsFalse(ContentNaming.RemoveDiacritics); - Assert.IsFalse(ContentNaming.Lowercase); Assert.IsFalse(ContentNaming.Transliterate); Assert.IsFalse(ContentNaming.ApplyCustomReplacements); - Assert.AreEqual(28, ContentNaming.ReplacementSources.Length); - Assert.AreEqual(ContentNaming.ReplacementSources.Length, ContentNaming.ReplacementTargets.Length); - CollectionAssert.AllItemsAreUnique(ContentNaming.ReplacementSources); - - const string original = "Ärger Straße Ærø Œuvre Łódź Þór Ħobż"; - Assert.AreEqual(original, - new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, original)); - } - - [DataTestMethod] - [DataRow("Fähre Straße Übergröße", "Fahre Strasse Ubergrosse")] - [DataRow("Ægir Œuvre Øresund", "AEgir OEuvre Oresund")] - [DataRow("Łódź Đakovo", "Lodz Dakovo")] - [DataRow("Þór Ðóra", "THor Dora")] - [DataRow("Ħobż ıslak İstanbul", "Hobz islak Istanbul")] - [DataRow("IJssel Ŋŋ Ŧŧ ſ ŀĿ", "IJssel Nn Tt s lL")] - [DataRow("Őrült tükörfúrógép", "Orult tukorfurogep")] - [DataRow("Ärger Öl Über", "Arger Ol Uber")] - [DataRow("A\u0308rger O\u0308l U\u0308ber", "Arger Ol Uber")] - public void PreparedEuropeanReplacements_ComplementDiacriticRemovalWithoutAnyAscii(string input, string expected) - { - using var replacement = new Swindler(true, - () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); - using var diacritics = new Swindler(true, - () => ContentNaming.RemoveDiacritics, value => ContentNaming.RemoveDiacritics = value); - - Assert.AreEqual(expected, - new CharReplacementContentNamingProvider().GenerateNameFromDisplayName(null, input)); - } - - [TestMethod] - public void PreparedEuropeanReplacements_KeepOriginalExtensionAndEncodingProvider() - { - using var replacement = new Swindler(true, - () => ContentNaming.ApplyCustomReplacements, value => ContentNaming.ApplyCustomReplacements = value); - using var casing = new Swindler(true, - () => ContentNaming.Lowercase, value => ContentNaming.Lowercase = value); - - Assert.AreEqual("strasse.DÓC", - new CharReplacementContentNamingProvider().GenerateNameFromDisplayName("old.DÓC", "Straße.DÓC")); - Assert.AreEqual("Straße.DÓC", - new Underscore5FContentNamingProvider().GenerateNameFromDisplayName(null, "Straße.DÓC")); + Assert.AreEqual(0, ContentNaming.ReplacementSources.Length); + Assert.AreEqual(0, ContentNaming.ReplacementTargets.Length); } [DataTestMethod] diff --git a/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs index d53b3e117..765e953eb 100644 --- a/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs +++ b/src/Tests/SenseNet.ODataTests/ODataContentNamingFeatureTests.cs @@ -46,19 +46,6 @@ await ODataTestAsync(async () => }).ConfigureAwait(false); } - [DataTestMethod] - [DataRow(false, "Ægir Straße Łódź Árvíztűrő tükörfúrógép")] - [DataRow(true, "AEgir Strasse Lodz Arvizturo tukorfurogep")] - public async Task OD_POST_PreparedLatinMappingsPreserveDisplayName(bool enabled, string expectedName) - { - await ODataTestAsync(async () => - { - using var replacement = ConfigureOption(nameof(ContentNaming.ApplyCustomReplacements), enabled); - using var diacritics = ConfigureOption(nameof(ContentNaming.RemoveDiacritics), enabled); - await CreateFolderAsync("Ægir Straße Łódź Árvíztűrő tükörfúrógép", expectedName).ConfigureAwait(false); - }).ConfigureAwait(false); - } - [DataTestMethod] [DataRow(nameof(ContentNaming.RemoveDiacritics), "Straße ARVIZ")] [DataRow(nameof(ContentNaming.Lowercase), "straße árvíz")] From a9c4de4ded24c71e4e42e13e93cb0215a06c46b7 Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Thu, 5 Nov 2026 12:09:56 +0100 Subject: [PATCH 8/9] Prepare disabled European naming replacements in webapp configuration --- docs/content-naming-provider.md | 51 ++++++++++++++----- .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ .../appsettings.json | 20 ++++++++ 8 files changed, 177 insertions(+), 14 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 1b223f38b..59838b542 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -52,37 +52,57 @@ Transliteration itself removes many diacritics. To preserve accents, disable bot `RemoveDiacritics` and `Transliterate` and avoid custom replacements that remove them. A Settings veto disables its step; other enabled steps can still have overlapping effects. -For example, this configuration generates `arvizturo-tukorfurogep` from -`Árvíztűrő tükörfúrógép` while preserving the original stored `DisplayName`: +The Docker webapps include prepared replacement pairs in their base `appsettings.json` +files. All four switches are explicitly `false`; the prepared list has no effect until +`ApplyCustomReplacements` is enabled. These are configuration values, not built-in +fallbacks in the provider or the configuration reader: ```json { "sensenet": { "contentNaming": { - "RemoveDiacritics": true, - "Lowercase": true, - "Transliterate": true, - "ApplyCustomReplacements": true, + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", - "&", - " " + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " ], "ReplacementTargets": [ - "ae", - "and", - "-" + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" ] } } } ``` +The list covers common additional Latin letters used in European languages, plus +`ä -> ae`, `& -> and` and spaces to hyphens. It is a starting point to customize for +the repository's languages, not a guarantee of language-specific transliteration. +For example, German `ö -> oe` and `ü -> ue` can be added explicitly; they are omitted +so Hungarian `tükör` can still become `tukor` with `RemoveDiacritics`. +Ordinary decomposable accented letters are handled by `RemoveDiacritics`; +other scripts can be handled by the separate `Transliterate` option. + +With these prepared pairs, enable `ApplyCustomReplacements`, `RemoveDiacritics` +and `Lowercase` to generate `arvizturo-tukorfurogep` from +`Árvíztűrő tükörfúrógép`. `Transliterate` can remain `false`. +The original stored `DisplayName` is preserved. + The replacement arrays are paired by index. Entries are case-sensitive literal strings, not regular expressions, and can contain multiple characters. An empty target string deletes a match. Sources must be non-empty strings and both arrays must have the same length; invalid pairs throw when custom replacement is enabled. -Empty arrays perform no replacements. Use string values, not JSON null values. +If no replacement entries are loaded, the custom step does nothing. +Use string values, not JSON null values. Pairs run in order and may cascade: `a -> b`, then `b -> c`, turns `ab` into `cc`. Custom replacement runs first, so `ä -> ae` can override the usual `ä -> a`. @@ -90,7 +110,10 @@ For Docker, the four switches are environment variables such as `sensenet__contentNaming__Transliterate=true`. Array elements use consecutive, zero-based indexes, for example `sensenet__contentNaming__ReplacementSources__0=ä` and `sensenet__contentNaming__ReplacementTargets__0=ae`. Do not leave gaps: array -loading stops at the first missing index. The host package/image must contain +loading stops at the first missing index. Configuration providers merge array +entries by index: a shorter array in a later provider does not clear remaining +entries from the base configuration. Edit the complete paired list in the base +`appsettings.json` when removing prepared pairs. The host package/image must contain these features; changing host configuration or replacement arrays requires a restart. ### Runtime Settings overrides diff --git a/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json index 31520ce3d..76774a538 100644 --- a/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json @@ -18,6 +18,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json index 6adecb8d9..81d66b858 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json index 62d9318de..0c997ba51 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json index fe3930bec..ad0a8c895 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json index 08cda6c22..a128bd174 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json index 467517aa8..dec10a3d4 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json index b78211dd3..4027d4543 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json @@ -21,6 +21,26 @@ }, "AllowedHosts": "*", "sensenet": { + "contentNaming": { + "RemoveDiacritics": false, + "Lowercase": false, + "Transliterate": false, + "ApplyCustomReplacements": false, + "ReplacementSources": [ + "ä", "Ä", "ß", "ẞ", + "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", + "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", + "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", + "&", " " + ], + "ReplacementTargets": [ + "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "o", "O", "l", "L", + "d", "D", "d", "D", "th", "TH", "h", "H", + "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", + "and", "-" + ] + }, "authentication": { "authServerType": "IdentityServer", "authority": "https://localhost:44311", From 63d3c066b9c3c111966d92bc7d0b2e9d6435bc41 Mon Sep 17 00:00:00 2001 From: VargaJoe Date: Thu, 5 Nov 2026 15:31:18 +0100 Subject: [PATCH 9/9] Prepare German umlaut replacement pairs in default configuration --- docs/content-naming-provider.md | 43 +++++++++++++------ .../appsettings.json | 4 +- .../appsettings.json | 4 +- .../appsettings.json | 4 +- .../appsettings.json | 4 +- .../appsettings.json | 4 +- .../appsettings.json | 4 +- .../appsettings.json | 4 +- 8 files changed, 44 insertions(+), 27 deletions(-) diff --git a/docs/content-naming-provider.md b/docs/content-naming-provider.md index 59838b542..7c3c66b81 100644 --- a/docs/content-naming-provider.md +++ b/docs/content-naming-provider.md @@ -66,14 +66,14 @@ fallbacks in the provider or the configuration reader: "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", @@ -84,17 +84,34 @@ fallbacks in the provider or the configuration reader: } ``` -The list covers common additional Latin letters used in European languages, plus -`ä -> ae`, `& -> and` and spaces to hyphens. It is a starting point to customize for -the repository's languages, not a guarantee of language-specific transliteration. -For example, German `ö -> oe` and `ü -> ue` can be added explicitly; they are omitted -so Hungarian `tükör` can still become `tukor` with `RemoveDiacritics`. -Ordinary decomposable accented letters are handled by `RemoveDiacritics`; -other scripts can be handled by the separate `Transliterate` option. - -With these prepared pairs, enable `ApplyCustomReplacements`, `RemoveDiacritics` -and `Lowercase` to generate `arvizturo-tukorfurogep` from -`Árvíztűrő tükörfúrógép`. `Transliterate` can remain `false`. +The list covers common additional Latin letters used in European languages and +includes German-style `ä -> ae`, `ö -> oe`, `ü -> ue`, `ß -> ss`, with uppercase +counterparts, plus `& -> and` and spaces to hyphens. It is a starting point to +customize for the repository's languages, not automatic language detection. +Every configured pair applies to every input when custom replacement is enabled. + +- For Hungarian text, enable `RemoveDiacritics` and leave + `ApplyCustomReplacements` disabled. No custom Hungarian accent pairs are needed. +- For German text, enable `ApplyCustomReplacements`. `RemoveDiacritics` can also + be enabled for other decomposable accents: custom replacement runs first, so + the German `ae`/`oe`/`ue` results are preserved. +- For mixed-language content, the operator chooses the switches and customizes + the list. For example, the prepared `ö -> oe` and `ü -> ue` pairs also change + Hungarian `tükör` to `tuekoer` when custom replacement is enabled. + +With only `ApplyCustomReplacements` enabled, `Fähre & Straße schön kühl` +becomes `Faehre-and-Strasse-schoen-kuehl`. The uppercase input `ÄÖÜẞ` becomes +`AEOEUESS` with the prepared pairs. + +With `RemoveDiacritics` and `Lowercase` enabled and custom replacement disabled, +`Árvíztűrő tükörfúrógép` becomes `arvizturo tukorfurogep`. Spaces remain under the +default invalid-character pattern; the existing `InvalidNameCharsPattern` and +`ReplacementChar` can be configured to replace whitespace with hyphens separately. +If custom replacement is enabled as well, the prepared German pairs instead +produce `arvizturo-tuekoerfurogep`. + +`RemoveDiacritics` alone maps `ä/ö/ü` to `a/o/u`; it does not change `ß` to `ss`. +Other scripts can be handled by the separate `Transliterate` option. The original stored `DisplayName` is preserved. The replacement arrays are paired by index. Entries are case-sensitive literal diff --git a/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json index 76774a538..8301a58b6 100644 --- a/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.InMem.TokenAuth/appsettings.json @@ -24,14 +24,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json index 81d66b858..bc181f062 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.LocalAuth/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json index 0c997ba51..ac3c85ac6 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth.Preview/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json index ad0a8c895..8410c5935 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.SearchService.TokenAuth/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json index a128bd174..efa616bff 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.NLB/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json index dec10a3d4..7ad392f50 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth.Preview/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L", diff --git a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json index 4027d4543..c58a3176a 100644 --- a/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json +++ b/src/WebApps/SnWebApplication.Api.Sql.TokenAuth/appsettings.json @@ -27,14 +27,14 @@ "Transliterate": false, "ApplyCustomReplacements": false, "ReplacementSources": [ - "ä", "Ä", "ß", "ẞ", + "ä", "Ä", "ö", "Ö", "ü", "Ü", "ß", "ẞ", "æ", "Æ", "œ", "Œ", "ø", "Ø", "ł", "Ł", "đ", "Đ", "ð", "Ð", "þ", "Þ", "ħ", "Ħ", "ı", "ij", "IJ", "ŋ", "Ŋ", "ŧ", "Ŧ", "ſ", "ŀ", "Ŀ", "&", " " ], "ReplacementTargets": [ - "ae", "AE", "ss", "SS", + "ae", "AE", "oe", "OE", "ue", "UE", "ss", "SS", "ae", "AE", "oe", "OE", "o", "O", "l", "L", "d", "D", "d", "D", "th", "TH", "h", "H", "i", "ij", "IJ", "n", "N", "t", "T", "s", "l", "L",