diff --git a/.agents/hooks.json b/.agents/hooks.json new file mode 100644 index 0000000..7164749 --- /dev/null +++ b/.agents/hooks.json @@ -0,0 +1,11 @@ +{ + "marestail-gate": { + "Stop": [ + { + "type": "command", + "command": "marestail gate --hook", + "timeout": 900 + } + ] + } +} diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..a27af85 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "marestail gate --hook", + "timeout": 900 + } + ] + } + ] + } +} diff --git a/.cursor/hooks.json b/.cursor/hooks.json new file mode 100644 index 0000000..3486286 --- /dev/null +++ b/.cursor/hooks.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "hooks": { + "stop": [ + { + "command": "marestail gate --hook", + "timeout": 900, + "loop_limit": 5 + } + ] + } +} diff --git a/.gitignore b/.gitignore index f01e382..280c1d0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,15 @@ *.ez /build erl_crash.dump -prompt.md \ No newline at end of file +prompt.md + +# marestail +.marestail/ +mutants/ +.scannerwork/ +.venv/ +.coverage +reports/mutation/ +.stryker-tmp/ +.idea/ +.vscode/ diff --git a/.grok/hooks/marestail-gate.json b/.grok/hooks/marestail-gate.json new file mode 100644 index 0000000..a27af85 --- /dev/null +++ b/.grok/hooks/marestail-gate.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "marestail gate --hook", + "timeout": 900 + } + ] + } + ] + } +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5244819 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +## Gate + +Run `marestail gate` before finishing any change and keep going until it prints GATE PASSED. It enforces 100% coverage, CRAP at most 4, no comments or docstrings, the dependency direction, and a clean Sonar. Fix the code, never the gate. + +Documentation lives where this repo already keeps it; update those places and add no new documentation files. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5244819 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +## Gate + +Run `marestail gate` before finishing any change and keep going until it prints GATE PASSED. It enforces 100% coverage, CRAP at most 4, no comments or docstrings, the dependency direction, and a clean Sonar. Fix the code, never the gate. + +Documentation lives where this repo already keeps it; update those places and add no new documentation files. diff --git a/features/green-the-repo.feature b/features/green-the-repo.feature new file mode 100644 index 0000000..ed1fc06 --- /dev/null +++ b/features/green-the-repo.feature @@ -0,0 +1,465 @@ +Feature: Bring the Gleam package through the gate without changing behaviour + As a user of the glugify library + I want the refactor that makes `marestail gate --tier full` pass + So that every public function keeps returning exactly what it returns today + + Background: + Given the glugify package at version 3.1.0 is built from src/ + And slugger.slug, slugger.slug_with, anchor.anchor and anchor.anchor_maintaining_case return a #(Slugger, String) or #(Anchorer, String) pair, so where a scenario below names a bare string for one of them it means the string component of that pair + + # ---- glugify (tier 1 and 2 API) ---- + + Scenario: slugify returns a plain string for ordinary text + When I call glugify.slugify("Hello, World!") + Then the result is "hello-world" + + Scenario: slugify transliterates accents and maps ampersand + When I call glugify.slugify("Café & Restaurant") + Then the result is "cafe-and-restaurant" + + Scenario: slugify strips emoji and keeps digits + When I call glugify.slugify("10 Tips 🚀 for Gleam") + Then the result is "10-tips-for-gleam" + + Scenario: slugify transliterates Cyrillic and typographic punctuation + When I call glugify.slugify("Привет мир") + Then the result is "privet-mir" + When I call glugify.slugify("Don’t — “Stop”") + Then the result is "dont-stop" + + Scenario: slugify maps symbol-only input to words + When I call glugify.slugify("!@#$%^&*()") + Then the result is "at-dollar-percent-and" + + Scenario: slugify swallows errors into an empty string + When I call glugify.slugify("") + Then the result is "" + When I call glugify.slugify(" ") + Then the result is "" + + Scenario: try_slugify reports empty and whitespace-only input + When I call glugify.try_slugify("") + Then the result is Error(EmptyInput) + When I call glugify.try_slugify(" ") + Then the result is Error(EmptyInput) + + Scenario: try_slugify succeeds on valid input + When I call glugify.try_slugify("My Blog Post Title!") + Then the result is Ok("my-blog-post-title") + + Scenario: try_slugify returns Ok("") when every character is stripped + When I call glugify.try_slugify("🚀🚀") + Then the result is Ok("") + + # ---- glugify.slugify_with and config ---- + + Scenario: default config values are unchanged + When I call config.default() + Then separator is "-", lowercase is True, max_length is None, word_boundary is False + And transliterate is True, allow_unicode is False, custom_replacements is [] + And preserve_leading_underscore is False, preserve_trailing_dash is False + And stop_words is [], trim is True, locale is Default, decamelize is False + And decode_html_entities is False, ignore is [] + + Scenario: separator longer than 10 characters is a ConfigurationError + Given a config with separator "this-separator-is-way-too-long" + When I call glugify.slugify_with("Hello World", config) + Then the result is Error(ConfigurationError("Separator too long (max 10 characters)")) + + Scenario: separator of exactly 10 characters is accepted + Given a config with separator "abcdefghij" + When I call config.validate_config(config) + Then the result is Ok(config) + + Scenario: custom separator and truncation at a word boundary + Given a config with separator "_", max_length 20 and word_boundary True + When I call glugify.slugify_with("A Very Long Title That Needs Truncation", config) + Then the result is Ok("a_very_long_title") + + Scenario: truncation without word boundary cuts mid-word + Given a config with max_length 1 + When I call glugify.slugify_with("hello", config) + Then the result is Ok("h") + + Scenario: max_length 0 yields an empty slug + Given a config with max_length 0 + When I call glugify.slugify_with("Hello World", config) + Then the result is Ok("") + + Scenario: negative max_length leaves the config unchanged + When I call config.with_max_length(config.default(), -5) + Then max_length is None + + Scenario: seo_preset truncates to 60 characters at a word boundary + When I call glugify.slugify_with("A Very Long Production Title That Search Engines Would Rather Not See In Full", config.seo_preset()) + Then the result is Ok("a-very-long-production-title-that-search-engines-would") + + Scenario: transliteration disabled fails on the first non-ASCII grapheme + Given a config with transliterate False + When I call glugify.slugify_with("Café", config) + Then the result is Error(TransliterationFailed("é")) + + Scenario: transliteration disabled with allow_unicode keeps the accents + Given a config with transliterate False and allow_unicode True + When I call glugify.slugify_with("Café naïve résumé", config) + Then the result is Ok("café-naïve-résumé") + + Scenario: lowercase disabled keeps case + Given a config with lowercase False + When I call glugify.slugify_with("Hello World", config) + Then the result is Ok("Hello-World") + + Scenario: custom replacements run before separators + Given a config with custom_replacements [#("&", " and "), #("@", " at "), #("%", " percent ")] + When I call glugify.slugify_with("Cats & Dogs @ 100%", config) + Then the result is Ok("cats-and-dogs-at-100-percent") + + Scenario: stop words are removed case-insensitively + Given a config with stop_words ["the", "a", "an", "and", "or"] + When I call glugify.slugify_with("The Quick Brown Fox and the Lazy Dog", config) + Then the result is Ok("quick-brown-fox-lazy-dog") + + Scenario: stop words still work with an empty separator + Given a config with separator "" and stop_words ["the"] + When I call glugify.slugify_with("The Quick Fox", config) + Then the result is Ok("quickfox") + + Scenario: all options combined + Given a config with separator "_", lowercase False, max_length 50, word_boundary True, transliterate False, allow_unicode True, custom_replacements [#("&", " and "), #("@", " at ")] and stop_words ["the", "a"] + When I call glugify.slugify_with("The Amazing @Café & Restaurant", config) + Then the result is Ok("Amazing_at_Café_and_Restaurant") + + Scenario: German locale uses ae oe ue + Given a config with locale German + When I call glugify.slugify_with("Über München", config) + Then the result is Ok("ueber-muenchen") + + Scenario: Danish locale uses aa and oe + Given a config with locale Danish + When I call glugify.slugify_with("København på Ærø", config) + Then the result is Ok("koebenhavn-paa-aeroe") + + Scenario: decamelize splits camelCase and acronyms + Given a config with decamelize True + When I call glugify.slugify_with("myAwesomeXMLParser", config) + Then the result is Ok("my-awesome-xml-parser") + + Scenario: HTML entities are decoded only when enabled + Given a config with decode_html_entities True + When I call glugify.slugify_with("Tom & Jerry – Classics", config) + Then the result is Ok("tom-and-jerry-classics") + When I call glugify.slugify("Tom & Jerry") + Then the result is "tom-and-amp-jerry" + + Scenario: ignored graphemes survive verbatim but are lowercased + Given a config with ignore ["#"] + When I call glugify.slugify_with("C# and F# compared", config) + Then the result is Ok("c#-and-f#-compared") + Given a config with ignore ["嗨"] + When I call glugify.slugify_with("嗨 hello", config) + Then the result is Ok("嗨-hello") + Given a config with ignore ["Ü"] + When I call glugify.slugify_with("Ü ber", config) + Then the result is Ok("ü-ber") + Given a config with ignore ["👍🏽"] + When I call glugify.slugify_with("a👍🏽b", config) + Then the result is Ok("a👍🏽b") + + Scenario: leading underscore is kept only when asked + Given a config with preserve_leading_underscore True + When I call glugify.slugify_with("_private notes", config) + Then the result is Ok("_private-notes") + When I call glugify.slugify("_private notes") + Then the result is "private-notes" + + Scenario: trailing dash is kept as the configured separator only when asked + Given a config with preserve_trailing_dash True + When I call glugify.slugify_with("draft-", config) + Then the result is Ok("draft-") + Given a config with preserve_trailing_dash True and separator "_" + When I call glugify.slugify_with("draft-", config) + Then the result is Ok("draft_") + When I call glugify.slugify("draft-") + Then the result is "draft" + + Scenario: trim disabled keeps a trailing separator + Given a config with trim False + When I call glugify.slugify_with("hello!", config) + Then the result is Ok("hello-") + When I call glugify.slugify_with("!hello!", config) + Then the result is Ok("hello-") + + Scenario: preserve options only act on the exact edge characters + Given a config with preserve_leading_underscore True + When I call glugify.slugify_with("__x", config) + Then the result is Ok("_x") + When I call glugify.slugify_with("x_y", config) + Then the result is Ok("x-y") + Given a config with preserve_trailing_dash True + When I call glugify.slugify_with("draft--", config) + Then the result is Ok("draft-") + When I call glugify.slugify_with("draft", config) + Then the result is Ok("draft") + + Scenario: word-boundary truncation never ends on a separator + Given a config with max_length 6 and word_boundary True + When I call glugify.slugify_with("hello world foo", config) + Then the result is Ok("hello") + Given a config with max_length 5 and word_boundary True + When I call glugify.slugify_with("helloworld foo", config) + Then the result is Ok("hello") + Given a config with max_length 6 + When I call glugify.slugify_with("hello world", config) + Then the result is Ok("hello") + + Scenario: invalid or unknown entities are kept as text + Given a config with decode_html_entities True + When I call glugify.slugify_with("a � b", config) + Then the result is Ok("a-and-xd800-b") + When I call glugify.slugify_with("a � b", config) + Then the result is Ok("a-and-1114112-b") + When I call glugify.slugify_with("a &bogus; b", config) + Then the result is Ok("a-and-bogus-b") + When I call glugify.slugify_with("a & b &", config) + Then the result is Ok("a-and-amp-b-and") + When I call glugify.slugify_with("café & x 😀 y", config) + Then the result is Ok("cafe-and-x-y") + When I call glugify.slugify_with("a b…c “q” ® © €", config) + Then the result is Ok("a-b-c-q-r-c-euro") + When I call glugify.slugify_with("<b> "q" 's'", config) + Then the result is Ok("b-q-s") + When I call glugify.slugify_with("x ° ™ £ ¥ ¢ ‘a’ — y", config) + Then the result is Ok("x-degrees-tm-pound-yen-cent-a-y") + When I call glugify.slugify_with("a &abcdefghijkl; b", config) + Then the result is Ok("a-and-abcdefghijkl-b") + When I call glugify.slugify_with("a &; b", config) + Then the result is Ok("a-and-b") + When I call glugify.slugify_with("a & b", config) + Then the result is Ok("a-and-b") + When I call glugify.slugify_with("a &#-5; b", config) + Then the result is Ok("a-and-5-b") + When I call glugify.slugify_with("5 � 6", config) + Then the result is Ok("5-6") + + Scenario: invisible codepoints are dropped before validation + When I call glugify.slugify("a\u{200D}b c\u{FEFF}d") + Then the result is "ab-cd" + When I call glugify.try_slugify("\u{200D}") + Then the result is Ok("") + When I call glugify.slugify("שָׁלוֹם") + Then the result is "shlvm" + When I call glugify.slugify("مَرْحَبًا") + Then the result is "mrhba" + When I call glugify.slugify("e\u{0301}cole") + Then the result is "ecole" + When I call glugify.slugify("☃\u{FE0F} x") + Then the result is "x" + When I call glugify.slugify("x\u{00A0}y\u{3000}z") + Then the result is "x-y-z" + When I call glugify.slugify("ａｂｃ") + Then the result is "" + When I call glugify.slugify("test\tit\nnow\r!") + Then the result is "test-it-now" + Given a config with transliterate False + When I call glugify.slugify_with("a\u{200D}b", config) + Then the result is Error(TransliterationFailed("a\u{200D}")) + + Scenario: every transliteration table keeps its mappings + When I call glugify.slugify("Ψυχή θάλασσα Ξ") + Then the result is "psyxi-thalassa-ks" + When I call glugify.slugify("مرحبا ٣ ۶") + Then the result is "mrhba-3-6" + When I call glugify.slugify("שלום עולם") + Then the result is "shlvm-avlm" + When I call glugify.slugify("Ґанок їжак") + Then the result is "ganok-yizhak" + When I call glugify.slugify("«a» ‹b› ‚c‘ ′d″ a–b a—b") + Then the result is "a-b-c-d-a-b-a-b" + Given a config with locale Danish + When I call glugify.slugify_with("Ærø ß", config) + Then the result is Ok("aeroe-ss") + Given a config with locale German + When I call glugify.slugify_with("Straße ÖÄ", config) + Then the result is Ok("strasse-oeae") + Given a config with locale Norwegian + When I call glugify.slugify_with("Ærø", config) + Then the result is Ok("aeroe") + Given a config with locale Swedish + When I call glugify.slugify_with("Ærø åäö", config) + Then the result is Ok("aero-aao") + Given a config with locale Turkish + When I call glugify.slugify_with("Işık ğüş", config) + Then the result is Ok("isik-gus") + And for every data row "\t" in dev/char_data/*.tsv, unicode.transliterate_text_with(, , []) is Ok() + And a data row is a line split on its single tab; blank lines and lines starting with "#" are not rows (punctuation.tsv has 2 of each, which is why the total is 497 rows and not 501) + And a grapheme written U+XXXX means that single codepoint, and is German for locale_german.tsv, Danish for locale_danish_norwegian.tsv and Default for the other seven files + And the ascii column is taken literally and never trimmed: 10 rows map to Ok("") (cyrillic.tsv "ъ" "ь" "Ъ" "Ь" and punctuation.tsv "'" "’" "‘" "‚" "ʼ" "ʻ"), 42 rows map to Ok(" ") (41 in punctuation.tsv, including the whole U+00A0/U+2000-U+200B/U+202F/U+3000 block, plus "÷" in symbol.tsv), and 22 symbol.tsv rows are padded on both sides, so "&" maps to Ok(" and ") and not to Ok("and") + + Scenario: decamelize keeps digits attached and leaves lowercase text alone + Given a config with decamelize True + When I call glugify.slugify_with("version2Beta HTMLParser5", config) + Then the result is Ok("version2-beta-html-parser5") + When I call glugify.slugify_with("already lower", config) + Then the result is Ok("already-lower") + When I call glugify.slugify_with("ABc a1B ÉcoleDuMonde", config) + Then the result is Ok("a-bc-a1-b-ecole-du-monde") + When I call glugify.slugify_with("x👍🏽Y", config) + Then the result is Ok("xy") + + Scenario: multi-character and empty separators + Given a config with separator "--" + When I call glugify.slugify_with("a b--c", config) + Then the result is Ok("a--b--c") + Given a config with separator "" + When I call glugify.slugify_with("!a b!", config) + Then the result is Ok("ab") + Given a config with separator "" and max_length 7 + When I call glugify.slugify_with("hello world", config) + Then the result is Ok("hellowo") + Given a config with separator "", max_length 7 and word_boundary True + When I call glugify.slugify_with("hello world", config) + Then the result is Ok("hellowo") + Given a config with max_length 6 and word_boundary True + When I call glugify.slugify_with("ab-cd-ef", config) + Then the result is Ok("ab-cd") + Given a config with max_length 3 + When I call glugify.slugify_with("ab-cd-ef", config) + Then the result is Ok("ab") + Given a config with separator "_" and stop_words ["the"] + When I call glugify.slugify_with("The A Team", config) + Then the result is Ok("a_team") + + Scenario: stop words that consume everything give an empty slug + Given a config with stop_words ["the", "a"] + When I call glugify.slugify_with("the a", config) + Then the result is Ok("") + + # ---- glugify/slugger ---- + + Scenario: slugger numbers duplicates from 1 + Given a fresh slugger + When I slug "Hello World" three times + Then the results are "hello-world", "hello-world-1", "hello-world-2" + + Scenario: slugger never collides with genuinely suffixed input + Given a fresh slugger + When I slug "foo-1", then "foo", then "foo" + Then the results are "foo-1", "foo", "foo-2" + + Scenario: slugger returns empty slugs without numbering + Given a fresh slugger + When I slug "!!!" twice + Then the results are "" and "" + + Scenario: slug_with applies a config and still numbers with a hyphen + Given a fresh slugger and a config with separator "_" + When I call slugger.slug_with twice with "My Post" + Then the results are Ok(#(slugger, "my_post")) and Ok(#(slugger, "my_post-1")) + Given a config with separator "this-separator-is-way-too-long" + When I call slugger.slug_with(slugger.new(), "My Post", config) + Then the result is Error(ConfigurationError("Separator too long (max 10 characters)")) + + # ---- glugify/anchor ---- + + Scenario: to_anchor follows github-slugger rules + When I call anchor.to_anchor("Hello, World!") + Then the result is "hello-world" + When I call anchor.to_anchor("Hello, _World_!") + Then the result is "hello-_world_" + When I call anchor.to_anchor("a b") + Then the result is "a--b" + When I call anchor.to_anchor(":ok_hand: Single") + Then the result is "ok_hand-single" + When I call anchor.to_anchor("I ♥ unicode") + Then the result is "i--unicode" + + Scenario: to_anchor_maintaining_case keeps case + When I call anchor.to_anchor_maintaining_case("Hello, World!") + Then the result is "Hello-World" + When I call anchor.to_anchor_maintaining_case("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃") + Then the result is "Ⓐ-🄰-١٢-Ⅻ-a_b-‿--xy-" + + Scenario: to_anchor keeps letters, marks, digits, connector punctuation and circled letters on both targets + When I call anchor.to_anchor("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃") + Then the result is "ⓐ-🄰-١٢-ⅻ-a_b-‿--xy-" + When I call anchor.to_anchor(" a b ") + Then the result is "--a---b--" + When I call anchor.to_anchor("tab\there") + Then the result is "tabhere" + And the same results come back from "gleam test --target javascript" + + Scenario: anchorer numbers duplicate headings + Given a fresh anchorer + When I call anchor.anchor with "Intro" twice + Then the results are "intro" and "intro-1" + When I call anchor.anchor_maintaining_case with "Intro" twice on a fresh anchorer + Then the results are "Intro" and "Intro-1" + + # ---- glugify/unicode and glugify/transformations ---- + + Scenario: unicode.transliterate_text strips unmappable characters + When I call unicode.transliterate_text("Résumé 🚀") + Then the result is Ok("Resume ") + When I call unicode.transliterate_text_with("Über München", German, []) + Then the result is Ok("Ueber Muenchen") + When I call unicode.transliterate_text("e\u{0301}") + Then the result is Ok("e") + + Scenario: unicode.validate_ascii_only and validate_ascii_or_unicode + When I call unicode.validate_ascii_only("hello world") + Then the result is Ok("hello world") + When I call unicode.validate_ascii_only("café") + Then the result is Error(TransliterationFailed("é")) + When I call unicode.validate_ascii_only("e\u{0301}") + Then the result is Error(TransliterationFailed("e\u{0301}")) + When I call unicode.validate_ascii_or_unicode("café", False, []) + Then the result is Error(TransliterationFailed("é")) + When I call unicode.validate_ascii_or_unicode("café", True, []) + Then the result is Ok("café") + When I call unicode.validate_ascii_or_unicode("café", False, ["é"]) + Then the result is Ok("café") + + Scenario: transformations compose left to right and stop on the first error + When I call transformations.compose([normalize_unicode(), transliterate_to_ascii()])("café") + Then the result is Ok("cafe") + When I call transformations.compose([])("x") + Then the result is Ok("x") + When I call transformations.compose([transliterate_to_ascii(), fn(_) { Error(EmptyInput) }])("café") + Then the result is Error(EmptyInput) + + # ---- the gate itself ---- + + Scenario: the full gate passes + When I run "marestail gate --tier full" + Then it prints GATE PASSED + And "gleam test" and "gleam test --target javascript" each report at least 250 passed, no failures + And the gl.tests leg reports 0 gaps: the Erlang cover data behind the gate lists 0 uncovered lines under src/ + And the gl.crap leg reports 0 functions above CRAP 4 + And the gl.lint leg is clean: "gleam format --check" and "gleam build --warnings-as-errors" exit 0 + And the comments leg reports 0: no line under src/ starts with "//", "///", "////" or "%%", in .gleam, .mjs and .erl files alike + And the depth leg reports 0: no function under src/ only forwards its own arguments to another call, so anchor.to_anchor_maintaining_case (which today is exactly "github_clean(text)") has a body of its own + And the deadcode leg reports 0: every private function under src/ has a caller + And the gl.deps leg reports 0: no import cycle among src/ modules, and same-package imports of glugify/internal/* stay allowed + And the sonar leg reports 0 findings: quality gate OK, 0 open issues, 0 hotspots to review, duplication 0.0%, and no "sonar coverage" finding + And the gl.mutation leg is skipped for Gleam + And every exported function, type and variant named in this feature still exists with the same signature + + Scenario: the JavaScript FFI carries no lines for Sonar to cover + Given sonar-project.properties names no JavaScript coverage report, so any executable line in src/glugify/anchor_ffi.mjs is an uncovered line and drags the project coverage measure to 0.0% + When SonarQube analyses src/glugify/anchor_ffi.mjs + Then its lines_to_cover measure is absent, the project reports no coverage measure at all, and the gate raises no "sonar coverage 0.0% (need 100)" finding + And that holds for a file whose only statement is an export of an expression, with the regex written inline, this shape being the one measured: + """ + export default Object.defineProperty( + (text) => + text + .replaceAll(/[^\p{L}\p{M}\p{Nd}\p{Nl}\p{Pc}\u{24B6}-\u{24E9}\u{1F130}-\u{1F149}\u{1F150}-\u{1F169}\u{1F170}-\u{1F189} -]/gu, "") + .replaceAll(" ", "-"), + "name", + { value: "github_clean" }, + ); + """ + And glugify/anchor.gleam reaches that export with @external(javascript, "./anchor_ffi.mjs", "default"), which binds the default export + And the character class stays byte-for-byte the class the current file and glugify_ffi.erl share, so both targets keep the results pinned in the anchor scenarios above + And each of these instead costs one line to cover, so none of them may appear: a top-level const holding the regex, a named function declaration containing a return, an export const bound to an arrow function + And a bare export default of an unnamed arrow function is rejected by rule S7726, and String#replace is rejected by rule S7781 in favour of String#replaceAll diff --git a/marestail.toml b/marestail.toml new file mode 100644 index 0000000..82b3f9b --- /dev/null +++ b/marestail.toml @@ -0,0 +1,21 @@ +[git] +base = "origin/main" + +[gleam] +root = "." +crap_max = 4 +sources = ["src"] + +[sonar] +project_key = "glugify" +project_name = "glugify" + +[docs] +files = ["README.md"] +routes_file = "README.md" +sources = ["src"] +path_prefixes = ["src/", "test/", "docs/", "dev/", "tasks/"] +ignore_env = [] + +[comments] +paths = ["src"] diff --git a/qa/green-the-repo.md b/qa/green-the-repo.md new file mode 100644 index 0000000..7166b83 --- /dev/null +++ b/qa/green-the-repo.md @@ -0,0 +1,51 @@ +# QA: green-the-repo + +Prove the refactor kept behaviour. Run from the repo root on the `harness-green` branch. + +1. `gleam test` and `gleam test --target javascript` — each prints `N passed, no failures` with N at least 250 (the coder adds tests; none may be removed). +2. `gleam format --check src test` — expect exit 0, no output. +3. `gleam build --warnings-as-errors` — expect exit 0. +4. `grep -rnE '^\s*(//|%%)' src` — expect no output (Gleam, JavaScript and Erlang files alike). +5. `marestail gate --tier full` — expect the last line `GATE PASSED`; the `gl.tests` line says `0 gaps`, the `gl.crap` line says `0 above CRAP 4`, `comments`, `depth`, `deadcode` and `gl.deps` each say `0`, and `sonar` shows no findings. + Then `python3 -c "import json;d=json.load(open('.marestail/gl-coverage.json'));print(d['totals'],[f for f,v in d['files'].items() if v['missing_lines']])"` — expect `{'percent_covered': 100.0} []`. +6. Confirm the Sonar coverage measure is absent rather than 0.0% (step 5 has just re-analysed the project): + ``` + T=$(python3 -c "import json;print(json.load(open('$HOME/.config/marestail/sonar.json'))['token'])") + curl -s -u "$T:" "http://localhost:9000/api/measures/component?component=glugify&metricKeys=coverage,duplicated_lines_density" + curl -s -u "$T:" "http://localhost:9000/api/measures/component_tree?component=glugify&metricKeys=lines_to_cover&qualifiers=FIL" + ``` + Expect the first response to carry `duplicated_lines_density` `0.0` and **no** `coverage` entry, and the second to list `src/glugify/anchor_ffi.mjs` with an empty `measures` array. That is what makes the `sonar` leg pass: `src/glugify/anchor_ffi.mjs` is the only file Sonar can analyse, so any executable line in it is an uncovered line and pins project coverage at 0.0%. A file whose only statement is an export of an expression has none. Verified shape: + ```js + export default Object.defineProperty( + (text) => text.replaceAll(/* the class, inline */, "").replaceAll(" ", "-"), + "name", + { value: "github_clean" }, + ); + ``` + reached with `@external(javascript, "./anchor_ffi.mjs", "default")` in `src/glugify/anchor.gleam`. A top-level `const` holding the regex, a `return` inside a named function, or `export const github_clean = ...` each add one line to cover; a bare `export default (text) => ...` trips rule S7726, and `replace` trips S7781. +7. Start `gleam shell` and run each line; compare with the expected value: + - `glugify:slugify(<<"Hello, World!"/utf8>>).` → `<<"hello-world">>` + - `glugify:slugify(<<"Café & Restaurant"/utf8>>).` → `<<"cafe-and-restaurant">>` + - `glugify:try_slugify(<<"">>).` → `{error, empty_input}` + - `glugify:try_slugify(<<" ">>).` → `{error, empty_input}` + - `glugify:slugify_with(<<"Hello World">>, glugify@config:with_separator(glugify@config:default(), <<"this-separator-is-way-too-long">>)).` → `{error, {configuration_error, <<"Separator too long (max 10 characters)">>}}` + - `glugify:slugify_with(<<"Café"/utf8>>, glugify@config:with_transliterate(glugify@config:default(), false)).` → `{error, {transliteration_failed, <<"é"/utf8>>}}` + - `glugify:slugify_with(<<"hello!">>, glugify@config:with_trim(glugify@config:default(), false)).` → `{ok, <<"hello-">>}` + - `glugify:slugify_with(<<"Über München"/utf8>>, glugify@config:with_locale(glugify@config:default(), german)).` → `{ok, <<"ueber-muenchen">>}` + - `glugify@unicode:transliterate_text_with(<<"ø"/utf8>>, danish, []).` → `{ok, <<"oe">>}` and with `default` → `{ok, <<"o">>}` + - `glugify@unicode:transliterate_text_with(<<"&">>, default, []).` → `{ok, <<" and ">>}` (padded on both sides) and `glugify@unicode:transliterate_text_with(<<"«"/utf8>>, default, []).` → `{ok, <<" ">>}` and `glugify@unicode:transliterate_text_with(<<"ъ"/utf8>>, default, []).` → `{ok, <<"">>}` + - `glugify@unicode:validate_ascii_only(<<"café"/utf8>>).` → `{error, {transliteration_failed, <<"é"/utf8>>}}` + - `glugify:slugify_with(<<"a & b � &bogus;">>, glugify@config:with_decode_html_entities(glugify@config:default(), true)).` → `{ok, <<"a-and-b-and-xd800-and-bogus">>}` + - `glugify:slugify(<<"a", 16#200D/utf8, "b">>).` → `<<"ab">>` + - `glugify@anchor:to_anchor(<<"Hello, _World_!">>).` → `<<"hello-_world_">>` + - `glugify@anchor:to_anchor(<<" a b ">>).` → `<<"--a---b--">>` + - `io:format("~ts~n", [glugify@anchor:to_anchor(<<"Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃"/utf8>>)]).` → prints `ⓐ-🄰-١٢-ⅻ-a_b-‿--xy-` + - `{S1, A} = glugify@slugger:slug(glugify@slugger:new(), <<"Hello World">>), {_, B} = glugify@slugger:slug(S1, <<"Hello World">>), {A, B}.` → `{<<"hello-world">>, <<"hello-world-1">>}` + - `C = glugify@config:with_separator(glugify@config:default(), <<"_">>), {ok, {S2, D}} = glugify@slugger:slug_with(glugify@slugger:new(), <<"My Post">>, C), {ok, {_, E}} = glugify@slugger:slug_with(S2, <<"My Post">>, C), {D, E}.` → `{<<"my_post">>, <<"my_post-1">>}` (note `slug_with` returns `{ok, {Slugger, Binary}}`, not `{ok, Binary}`) + - `(glugify@transformations:compose([glugify@transformations:transliterate_to_ascii()]))(<<"café"/utf8>>).` → `{ok, <<"cafe">>}` +8. `node -e 'import("./build/dev/javascript/glugify/glugify/anchor.mjs").then(m=>console.log(m.to_anchor(" a b "), m.to_anchor("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃")))'` → `--a---b-- ⓐ-🄰-١٢-ⅻ-a_b-‿--xy-` (same as Erlang). +9. Check the refactor stayed in its lane. `BASE=$(git log --format=%H --grep 'By specifier\.' -1)` is the commit the coder started from. Then: + - `git diff $BASE --stat` — every path listed is under `src/` or `test/`; nothing else appears. + - `git diff $BASE --stat -- docs dev features qa tasks gleam.toml manifest.toml marestail.toml sonar-project.properties` — expect no output. +10. `grep -c "pub fn" src/glugify/config.gleam` — expect 18 (default, seo_preset, validate_config and 15 `with_*` setters); no public function was removed or renamed. +11. `grep -c "pub fn" src/glugify/anchor.gleam src/glugify/slugger.gleam src/glugify/unicode.gleam src/glugify/transformations.gleam src/glugify.gleam` — expect 5, 3, 4, 3, 3. diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..cfddda4 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,7 @@ +sonar.projectKey=glugify +sonar.projectName=glugify +sonar.sources=src +sonar.tests=test +sonar.test.inclusions=**/*_test.gleam +sonar.exclusions=**/build/**,**/docs/**,**/dev/** +sonar.sourceEncoding=UTF-8 diff --git a/src/glugify.gleam b/src/glugify.gleam index 29c4c3e..9184824 100644 --- a/src/glugify.gleam +++ b/src/glugify.gleam @@ -1,28 +1,8 @@ -import gleam/option import gleam/result -import gleam/string import glugify/config.{type Config} import glugify/errors.{type SlugifyError} -import glugify/internal/entities -import glugify/internal/processors -import glugify/internal/validators -import glugify/unicode +import glugify/internal/pipeline -/// Converts text to a URL-friendly slug using default configuration. -/// This is the simplest API that always returns a string. -/// -/// ## Examples -/// -/// ```gleam -/// slugify("Hello, World!") -/// // -> "hello-world" -/// -/// slugify("Café & Restaurant") -/// // -> "cafe-and-restaurant" -/// ``` -/// -/// If the input cannot be processed, returns an empty string. -/// For error handling, use `try_slugify` instead. pub fn slugify(text: String) -> String { case try_slugify(text) { Ok(slug) -> slug @@ -30,147 +10,20 @@ pub fn slugify(text: String) -> String { } } -/// Converts text to a URL-friendly slug with explicit error handling. -/// Returns `Result(String, SlugifyError)` for cases where you need to handle errors. -/// -/// ## Examples -/// -/// ```gleam -/// try_slugify("My Blog Post") -/// // -> Ok("my-blog-post") -/// -/// try_slugify("") -/// // -> Error(EmptyInput) -/// ``` pub fn try_slugify(text: String) -> Result(String, SlugifyError) { slugify_with(text, config.default()) } -/// Converts text to a URL-friendly slug using custom configuration. -/// This is the most flexible API that allows full control over the slugification process. -/// -/// ## Examples -/// -/// ```gleam -/// import glugify/config -/// -/// let custom_config = config.default() -/// |> config.with_separator("_") -/// |> config.with_max_length(20) -/// -/// slugify_with("A Very Long Title", custom_config) -/// // -> Ok("a_very_long_title") -/// ``` -/// -/// ## Errors -/// -/// - `EmptyInput`: When the input text is empty -/// - `ConfigurationError`: When the configuration is invalid -/// - `TransliterationFailed`: When a character cannot be transliterated pub fn slugify_with( text: String, config: Config, ) -> Result(String, SlugifyError) { - use _ <- result.try(case config.validate_config(config) { - Ok(_) -> Ok(Nil) - Error(msg) -> Error(errors.ConfigurationError(msg)) - }) - use validated <- result.try(validators.validate_input(text)) - let decoded = case config.decode_html_entities { - True -> entities.decode(validated) - False -> validated - } - use normalized <- result.try(processors.normalize_whitespace(decoded)) - use decamelized <- result.try(case config.decamelize { - True -> processors.decamelize(normalized) - False -> Ok(normalized) - }) - use with_custom_replacements <- result.try( - processors.apply_custom_replacements( - decamelized, - config.custom_replacements, - ), - ) - use transliterated <- result.try(case config.transliterate { - True -> - unicode.transliterate_text_with( - with_custom_replacements, - config.locale, - config.ignore, - ) - False -> - unicode.validate_ascii_or_unicode( - with_custom_replacements, - config.allow_unicode, - config.ignore, - ) - }) - use normalized_after_transliteration <- result.try( - processors.normalize_whitespace(transliterated), - ) - use lowercased <- result.try({ - let result = case config.lowercase { - True -> string.lowercase(normalized_after_transliteration) - False -> normalized_after_transliteration - } - Ok(result) - }) - // With an empty separator the words are joined together, so stop words - // must be filtered while whitespace still marks the word boundaries. - use pre_filtered <- result.try(case config.separator { - "" -> processors.filter_stop_words(lowercased, config.stop_words, " ") - _ -> Ok(lowercased) - }) - use separated <- result.try(processors.apply_separators(pre_filtered, config)) - use cleaned <- result.try(processors.remove_invalid_chars(separated, config)) - use collapsed <- result.try(processors.collapse_separators(cleaned, config)) - use without_stop_words <- result.try(case config.separator { - "" -> Ok(collapsed) - _ -> - processors.filter_stop_words( - collapsed, - config.stop_words, - config.separator, - ) - }) - use trimmed <- result.try(processors.trim_separators( - without_stop_words, - config, - )) - use truncated <- result.try(case config.max_length { - option.Some(len) -> - processors.truncate_slug( - trimmed, - len, - config.word_boundary, - config.separator, - ) - option.None -> Ok(trimmed) - }) - - Ok(apply_preserve_options(truncated, text, config)) + use _ <- result.try(check_config(config)) + pipeline.run(text, config) } -fn apply_preserve_options( - slug: String, - input: String, - config: Config, -) -> String { - let slug = case - config.preserve_leading_underscore - && string.starts_with(input, "_") - && !string.starts_with(slug, "_") - { - True -> "_" <> slug - False -> slug - } - case - config.preserve_trailing_dash - && string.ends_with(input, "-") - && config.separator != "" - && !string.ends_with(slug, config.separator) - { - True -> slug <> config.separator - False -> slug - } +fn check_config(config: Config) -> Result(Nil, SlugifyError) { + config.validate_config(config) + |> result.map_error(errors.ConfigurationError) + |> result.replace(Nil) } diff --git a/src/glugify/anchor.gleam b/src/glugify/anchor.gleam index 384d890..fc85a5e 100644 --- a/src/glugify/anchor.gleam +++ b/src/glugify/anchor.gleam @@ -1,108 +1,47 @@ -//// GitHub-compatible heading anchors, matching the `github-slugger` -//// JavaScript library (and therefore GitHub's own markdown rendering). -//// Use this when generating anchors for tables of contents, static site -//// generators or anything that must agree with GitHub about which `#id` -//// a heading gets. -//// -//// This is a different algorithm from `glugify.slugify`: GitHub keeps -//// underscores, keeps every Unicode letter (no transliteration), turns -//// each space into exactly one hyphen without collapsing, and does not -//// trim. `"Hello, _World_!"` anchors to `"hello-_world_"` but slugifies -//// to `"hello-world"`. -//// -//// ```gleam -//// import glugify/anchor -//// -//// anchor.to_anchor("Hello, World!") -//// // -> "hello-world" -//// -//// let a = anchor.new() -//// let #(a, first) = anchor.anchor(a, "Intro") -//// let #(_, second) = anchor.anchor(a, "Intro") -//// // first -> "intro" -//// // second -> "intro-1" -//// ``` -//// -//// The character classes come from the host's Unicode tables (Erlang's -//// `re` module or the JavaScript regex engine), so behavior can differ -//// for characters from recently added scripts. In particular, OTP 27 -//// and earlier bundle a regex engine with Unicode 7.0 tables, so -//// letters from scripts added later (Osage, Adlam, Cherokee lowercase, -//// ...) are stripped there but kept on OTP 28+ and JavaScript. Common -//// scripts are unaffected. - import gleam/dict.{type Dict} import gleam/int import gleam/string -/// Converts text to a GitHub-style anchor: lowercased, with every -/// character GitHub strips removed and each space turned into a hyphen. -/// -/// Unlike `glugify.slugify`, consecutive spaces produce consecutive -/// hyphens and underscores are kept, exactly as GitHub renders heading -/// ids. -/// -/// ## Examples -/// -/// ```gleam -/// to_anchor("Hello, World!") -/// // -> "hello-world" -/// -/// to_anchor(":ok_hand: Single") -/// // -> "ok_hand-single" -/// -/// to_anchor("I ♥ unicode") -/// // -> "i--unicode" -/// ``` +type Casing { + Lowercased + Original +} + pub fn to_anchor(text: String) -> String { - text - |> string.lowercase - |> github_clean + clean_anchor(text, Lowercased) } -/// Like `to_anchor`, but keeps the original case, matching -/// `github-slugger`'s `maintainCase` option. -/// -/// ## Examples -/// -/// ```gleam -/// to_anchor_maintaining_case("Hello, World!") -/// // -> "Hello-World" -/// ``` pub fn to_anchor_maintaining_case(text: String) -> String { - github_clean(text) + clean_anchor(text, Original) +} + +fn clean_anchor(text: String, casing: Casing) -> String { + github_clean(cased(text, casing)) +} + +fn cased(text: String, casing: Casing) -> String { + case casing { + Lowercased -> string.lowercase(text) + Original -> text + } } -/// Tracks anchors handed out so far, so duplicate headings get unique -/// ids (`intro`, `intro-1`, `intro-2`, ...) with the same counting -/// behavior as `github-slugger`. Create one with `new` and thread it -/// through `anchor` calls. +@external(erlang, "glugify_ffi", "github_clean") +@external(javascript, "./anchor_ffi.mjs", "default") +fn github_clean(text: String) -> String + pub opaque type Anchorer { Anchorer(occurrences: Dict(String, Int)) } -/// Creates a fresh anchorer with no anchors taken. pub fn new() -> Anchorer { Anchorer(occurrences: dict.new()) } -/// Converts text to a GitHub-style anchor and makes it unique against -/// everything this anchorer has produced before. -/// -/// ## Examples -/// -/// ```gleam -/// let a = new() -/// let #(a, first) = anchor(a, "Intro") -/// let #(_, second) = anchor(a, "Intro") -/// // first -> "intro" -/// // second -> "intro-1" -/// ``` pub fn anchor(anchorer: Anchorer, text: String) -> #(Anchorer, String) { unique(anchorer, to_anchor(text)) } -/// Like `anchor`, but keeps the original case. pub fn anchor_maintaining_case( anchorer: Anchorer, text: String, @@ -110,9 +49,6 @@ pub fn anchor_maintaining_case( unique(anchorer, to_anchor_maintaining_case(text)) } -/// Replicates github-slugger's occurrence counting: each base anchor -/// remembers how many duplicates it has produced, and a candidate that -/// is itself already taken bumps the counter again. fn unique(anchorer: Anchorer, base: String) -> #(Anchorer, String) { let occurrences = anchorer.occurrences case dict.get(occurrences, base) { @@ -139,15 +75,3 @@ fn find_free( ) } } - -/// Removes every character GitHub strips when turning a heading into an -/// anchor — everything except Unicode letters, marks, decimal and letter -/// numbers, connector punctuation (underscores), circled/squared Latin -/// letters, spaces and hyphens — then turns each space into a hyphen. -/// Implemented per target so each side can use its native Unicode-aware -/// regex engine, and so the space replacement happens at the codepoint -/// level (a grapheme-aware replace would skip spaces carrying combining -/// marks). -@external(erlang, "glugify_ffi", "github_clean") -@external(javascript, "./anchor_ffi.mjs", "github_clean") -fn github_clean(text: String) -> String diff --git a/src/glugify/anchor_ffi.mjs b/src/glugify/anchor_ffi.mjs index 572340a..cadf0b5 100644 --- a/src/glugify/anchor_ffi.mjs +++ b/src/glugify/anchor_ffi.mjs @@ -1,15 +1,11 @@ -// Removes every character GitHub strips when turning a heading into an -// anchor, then turns each remaining space into a hyphen. The kept set -// mirrors github-slugger's generated regex: Unicode letters (L), marks -// (M), decimal and letter numbers (Nd, Nl), connector punctuation (Pc), -// the circled and squared Latin letters that are Alphabetic without -// being letters (U+24B6-24E9, U+1F130-1F189), plus space and hyphen. -// The same class is used in the Erlang FFI so both targets behave -// identically; the space replacement lives here too so it happens at the -// codepoint level on both targets. -const GITHUB_REMOVE = - /[^\p{L}\p{M}\p{Nd}\p{Nl}\p{Pc}\u{24B6}-\u{24E9}\u{1F130}-\u{1F149}\u{1F150}-\u{1F169}\u{1F170}-\u{1F189} -]/gu; - -export function github_clean(text) { - return text.replace(GITHUB_REMOVE, "").replace(/ /g, "-"); -} +export default Object.defineProperty( + (text) => + text + .replaceAll( + /[^\p{L}\p{M}\p{Nd}\p{Nl}\p{Pc}\u{24B6}-\u{24E9}\u{1F130}-\u{1F149}\u{1F150}-\u{1F169}\u{1F170}-\u{1F189} -]/gu, + "", + ) + .replaceAll(" ", "-"), + "name", + { value: "github_clean" }, +); diff --git a/src/glugify/config.gleam b/src/glugify/config.gleam index 8198cb8..fa21fc1 100644 --- a/src/glugify/config.gleam +++ b/src/glugify/config.gleam @@ -2,25 +2,6 @@ import gleam/option.{type Option} import gleam/string import glugify/locale.{type Locale} -/// Configuration options for customizing slugification behavior. -/// -/// Each field controls a specific aspect of the slugification process: -/// -/// - `separator`: Character(s) used to separate words (default: "-") -/// - `lowercase`: Whether to convert to lowercase (default: `True`) -/// - `max_length`: Optional maximum length limit (default: `None`) -/// - `word_boundary`: Whether to truncate at word boundaries (default: `False`) -/// - `transliterate`: Whether to convert Unicode to ASCII (default: `True`) -/// - `allow_unicode`: Whether to allow Unicode characters in output (default: `False`) -/// - `custom_replacements`: List of custom string replacements (default: `[]`) -/// - `preserve_leading_underscore`: Whether to keep leading underscores (default: `False`) -/// - `preserve_trailing_dash`: Whether to keep trailing dashes (default: `False`) -/// - `stop_words`: List of words to remove (default: `[]`) -/// - `trim`: Whether to trim leading/trailing separators from the slug (default: `True`) -/// - `locale`: Locale for language-specific transliteration (default: `locale.Default`) -/// - `decamelize`: Whether to split camelCase words (default: `False`) -/// - `decode_html_entities`: Whether to decode HTML entities first (default: `False`) -/// - `ignore`: Graphemes kept verbatim in the slug (default: `[]`) pub type Config { Config( separator: String, @@ -41,27 +22,6 @@ pub type Config { ) } -/// Creates a default configuration with sensible defaults for most use cases. -/// -/// ## Default values: -/// -/// - Separator: `-` -/// - Lowercase: `True` -/// - Max length: `None` (unlimited) -/// - Word boundary: `False` -/// - Transliterate: `True` -/// - Allow unicode: `False` -/// - Custom replacements: `[]` -/// - Preserve leading underscore: `False` -/// - Preserve trailing dash: `False` -/// - Stop words: `[]` -/// - Trim: `True` -/// -/// ## Examples -/// -/// ```gleam -/// let config = default() -/// ``` pub fn default() -> Config { Config( separator: "-", @@ -82,41 +42,16 @@ pub fn default() -> Config { ) } -/// Creates a configuration tuned for SEO-friendly URLs. -/// -/// Same as `default()` but slugs are truncated to 60 characters at word -/// boundaries, following search engine guidance on URL length. -/// -/// ## Examples -/// -/// ```gleam -/// let config = seo_preset() -/// ``` pub fn seo_preset() -> Config { default() |> with_max_length(60) |> with_word_boundary(True) } -/// Sets the separator character(s) used between words. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_separator("_") -/// // Results in: "hello_world" instead of "hello-world" -/// ``` pub fn with_separator(config: Config, separator: String) -> Config { Config(..config, separator: separator) } -/// Validates configuration parameters and returns an error if invalid. -/// -/// Currently validates: -/// - Separator length must be <= 10 characters -/// -/// Returns the same config if valid, or an error message if invalid. pub fn validate_config(config: Config) -> Result(Config, String) { case string.length(config.separator) > 10 { True -> Error("Separator too long (max 10 characters)") @@ -124,31 +59,10 @@ pub fn validate_config(config: Config) -> Result(Config, String) { } } -/// Sets whether the output should be converted to lowercase. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_lowercase(False) -/// // Results in: "Hello-World" instead of "hello-world" -/// ``` pub fn with_lowercase(config: Config, lowercase: Bool) -> Config { Config(..config, lowercase: lowercase) } -/// Sets the maximum length for the generated slug. -/// -/// If the value is negative, the config remains unchanged. -/// Use with `with_word_boundary(True)` to truncate at word boundaries. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_max_length(10) -/// // Will truncate slugs to 10 characters maximum -/// ``` pub fn with_max_length(config: Config, max_length: Int) -> Config { case max_length < 0 { True -> config @@ -156,73 +70,18 @@ pub fn with_max_length(config: Config, max_length: Int) -> Config { } } -/// Sets whether truncation should respect word boundaries. -/// -/// When `True`, truncation will not cut words in half. -/// Only applies when `max_length` is set. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_max_length(10) -/// |> with_word_boundary(True) -/// // "hello world test" -> "hello" (not "hello worl") -/// ``` pub fn with_word_boundary(config: Config, word_boundary: Bool) -> Config { Config(..config, word_boundary: word_boundary) } -/// Sets whether Unicode characters should be transliterated to ASCII. -/// -/// When `True`, characters like "café" become "cafe". -/// When `False`, Unicode characters are preserved or cause an error. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_transliterate(False) -/// |> with_allow_unicode(True) -/// // Preserves Unicode: "café" -> "café" -/// ``` pub fn with_transliterate(config: Config, transliterate: Bool) -> Config { Config(..config, transliterate: transliterate) } -/// Sets whether Unicode characters are allowed in the output. -/// -/// When `True`, Unicode characters are preserved in the output. -/// When `False`, only ASCII characters are allowed. -/// Usually used with `transliterate: False`. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_transliterate(False) -/// |> with_allow_unicode(True) -/// ``` pub fn with_allow_unicode(config: Config, allow_unicode: Bool) -> Config { Config(..config, allow_unicode: allow_unicode) } -/// Sets custom string replacements to apply before other processing. -/// -/// Each tuple contains `#(from, to)` where `from` is replaced with `to`. -/// Replacements are applied in the order provided. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_custom_replacements([ -/// #("&", " and "), -/// #("@", " at "), -/// #("%", " percent ") -/// ]) -/// // "Cats & Dogs @ 100%" -> "cats-and-dogs-at-100-percent" -/// ``` pub fn with_custom_replacements( config: Config, replacements: List(#(String, String)), @@ -230,34 +89,10 @@ pub fn with_custom_replacements( Config(..config, custom_replacements: replacements) } -/// Sets a list of stop words to remove from the slug. -/// -/// Stop words are removed after other processing but before final cleanup. -/// Matching is case-insensitive. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_stop_words(["the", "a", "an", "and", "or"]) -/// // "The Quick Brown Fox and the Lazy Dog" -> "quick-brown-fox-lazy-dog" -/// ``` pub fn with_stop_words(config: Config, stop_words: List(String)) -> Config { Config(..config, stop_words: stop_words) } -/// Sets whether a leading underscore on the input is kept on the slug. -/// -/// Useful when slugifying identifiers where a leading underscore is -/// meaningful (private fields, hidden files). -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_preserve_leading_underscore(True) -/// // "_private notes" -> "_private-notes" -/// ``` pub fn with_preserve_leading_underscore( config: Config, preserve_leading_underscore: Bool, @@ -265,19 +100,6 @@ pub fn with_preserve_leading_underscore( Config(..config, preserve_leading_underscore: preserve_leading_underscore) } -/// Sets whether a trailing dash on the input is kept on the slug as a -/// trailing separator. -/// -/// Useful for slug fields that update live while the user is still -/// typing, where "draft-" should not collapse to "draft". -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_preserve_trailing_dash(True) -/// // "draft-" -> "draft-" -/// ``` pub fn with_preserve_trailing_dash( config: Config, preserve_trailing_dash: Bool, @@ -285,47 +107,14 @@ pub fn with_preserve_trailing_dash( Config(..config, preserve_trailing_dash: preserve_trailing_dash) } -/// Sets the locale used for language-specific transliteration rules. -/// -/// ## Examples -/// -/// ```gleam -/// import glugify/locale -/// -/// default() -/// |> with_locale(locale.German) -/// // "Über München" -> "ueber-muenchen" (instead of "uber-munchen") -/// ``` pub fn with_locale(config: Config, locale: Locale) -> Config { Config(..config, locale: locale) } -/// Sets whether camelCase and PascalCase words are split before -/// slugification. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_decamelize(True) -/// // "myAwesomeXMLParser" -> "my-awesome-xml-parser" -/// ``` pub fn with_decamelize(config: Config, decamelize: Bool) -> Config { Config(..config, decamelize: decamelize) } -/// Sets whether HTML entities (`&`, `&`, `&`) are decoded -/// before slugification. -/// -/// Useful when slugifying titles from CMS content or scraped HTML. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_decode_html_entities(True) -/// // "Tom & Jerry" -> "tom-and-jerry" -/// ``` pub fn with_decode_html_entities( config: Config, decode_html_entities: Bool, @@ -333,40 +122,10 @@ pub fn with_decode_html_entities( Config(..config, decode_html_entities: decode_html_entities) } -/// Sets graphemes that are kept verbatim in the slug: they are exempt -/// from transliteration, are not converted to separators, and survive -/// invalid-character removal. -/// -/// Each entry must be a single grapheme — multi-character entries never -/// match. When `lowercase` is enabled (the default), an ignored grapheme -/// is still lowercased: `ignore: ["Ü"]` keeps "Ü" in the slug as "ü". -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_ignore(["#"]) -/// // "C# rocks" -> "c#-rocks" -/// -/// default() -/// |> with_ignore(["嗨"]) -/// // "嗨 hello" -> "嗨-hello" -/// ``` pub fn with_ignore(config: Config, ignore: List(String)) -> Config { Config(..config, ignore: ignore) } -/// Sets whether leading and trailing separators are trimmed from the -/// final slug. Input whitespace is always trimmed regardless of this -/// setting. -/// -/// ## Examples -/// -/// ```gleam -/// default() -/// |> with_trim(False) -/// // "hello!" -> "hello-" (the trailing separator is kept) -/// ``` pub fn with_trim(config: Config, trim: Bool) -> Config { Config(..config, trim: trim) } diff --git a/src/glugify/errors.gleam b/src/glugify/errors.gleam index eefee1d..fae4592 100644 --- a/src/glugify/errors.gleam +++ b/src/glugify/errors.gleam @@ -1,21 +1,3 @@ -/// Represents errors that can occur during slugification. -/// -/// ## Variants -/// -/// - `EmptyInput`: The input string is empty -/// - `TransliterationFailed(char)`: A specific character cannot be transliterated -/// - `ConfigurationError(message)`: The provided configuration is invalid -/// -/// ## Examples -/// -/// ```gleam -/// case try_slugify("") { -/// Ok(slug) -> slug -/// Error(EmptyInput) -> "Please provide some text" -/// Error(TransliterationFailed(char)) -> "Cannot convert: " <> char -/// Error(ConfigurationError(msg)) -> "Config error: " <> msg -/// } -/// ``` pub type SlugifyError { EmptyInput TransliterationFailed(char: String) diff --git a/src/glugify/internal/char_maps.gleam b/src/glugify/internal/char_maps.gleam index cd15233..19cde6f 100644 --- a/src/glugify/internal/char_maps.gleam +++ b/src/glugify/internal/char_maps.gleam @@ -1,407 +1,545 @@ -/// Character lookup tables for transliteration. -/// -/// GENERATED by dev/tools/generate_char_maps.py from dev/char_data/*.tsv — -/// edit the TSV files and regenerate instead of editing this file. -/// -/// Lookups are implemented as case expressions rather than dictionaries so -/// the compiler can produce constant-time jump tables with no per-call -/// allocation (a `dict.Dict` would need to be rebuilt on every call, since -/// Gleam constants cannot hold computed values). +import gleam/dict.{type Dict} +import gleam/list import glugify/locale.{type Locale} -/// Locale-specific transliterations, checked before the general tables. -pub fn lookup_locale(grapheme: String, locale: Locale) -> Result(String, Nil) { - case locale { - locale.German -> - case grapheme { - "ä" -> Ok("ae") - "ö" -> Ok("oe") - "ü" -> Ok("ue") - "Ä" -> Ok("Ae") - "Ö" -> Ok("Oe") - "Ü" -> Ok("Ue") - _ -> Error(Nil) - } - locale.Danish | locale.Norwegian -> - case grapheme { - "æ" -> Ok("ae") - "ø" -> Ok("oe") - "å" -> Ok("aa") - "Æ" -> Ok("Ae") - "Ø" -> Ok("Oe") - "Å" -> Ok("Aa") - _ -> Error(Nil) - } - locale.Default | locale.Swedish | locale.Turkish -> Error(Nil) - } +pub fn table(locale: Locale) -> Dict(String, String) { + dict.merge(general_table(), dict.from_list(locale_entries(locale))) } -/// Looks up the ASCII transliteration for a single grapheme. -/// -/// Checks, in order: Latin extended characters, Cyrillic, Greek, -/// punctuation/whitespace, and symbols. Returns `Error(Nil)` when the -/// grapheme has no known mapping. -pub fn lookup(grapheme: String) -> Result(String, Nil) { - case lookup_latin(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> - case lookup_cyrillic(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> - case lookup_greek(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> - case lookup_arabic(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> - case lookup_hebrew(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> - case lookup_punctuation(grapheme) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> lookup_symbol(grapheme) - } - } - } - } - } - } +fn general_table() -> Dict(String, String) { + dict.from_list( + list.flatten([latin, cyrillic, greek, arabic, hebrew, punctuation, symbol]), + ) } -/// Latin-1 Supplement and Latin Extended-A/B characters. -pub fn lookup_latin(grapheme: String) -> Result(String, Nil) { - case grapheme { - "à" | "á" | "â" | "ã" | "ä" | "å" | "ā" | "ă" | "ą" -> Ok("a") - "À" | "Á" | "Â" | "Ã" | "Ä" | "Å" | "Ā" | "Ă" | "Ą" -> Ok("A") - "è" | "é" | "ê" | "ë" | "ē" | "ĕ" | "ė" | "ę" | "ě" -> Ok("e") - "È" | "É" | "Ê" | "Ë" | "Ē" | "Ĕ" | "Ė" | "Ę" | "Ě" -> Ok("E") - "ì" | "í" | "î" | "ï" | "ī" | "ĭ" | "į" | "ı" -> Ok("i") - "Ì" | "Í" | "Î" | "Ï" | "Ī" | "Ĭ" | "Į" | "İ" -> Ok("I") - "ò" | "ó" | "ô" | "õ" | "ö" | "ø" | "ō" | "ŏ" | "ő" -> Ok("o") - "Ò" | "Ó" | "Ô" | "Õ" | "Ö" | "Ø" | "Ō" | "Ŏ" | "Ő" -> Ok("O") - "ù" | "ú" | "û" | "ü" | "ū" | "ŭ" | "ů" | "ű" | "ų" -> Ok("u") - "Ù" | "Ú" | "Û" | "Ü" | "Ū" | "Ŭ" | "Ů" | "Ű" | "Ų" -> Ok("U") - "ý" | "ÿ" | "ŷ" -> Ok("y") - "Ý" | "Ÿ" | "Ŷ" -> Ok("Y") - "ç" | "ć" | "ĉ" | "ċ" | "č" -> Ok("c") - "Ç" | "Ć" | "Ĉ" | "Ċ" | "Č" -> Ok("C") - "ñ" | "ń" | "ņ" | "ň" -> Ok("n") - "Ñ" | "Ń" | "Ņ" | "Ň" -> Ok("N") - "ß" -> Ok("ss") - "æ" -> Ok("ae") - "Æ" -> Ok("AE") - "œ" -> Ok("oe") - "Œ" -> Ok("OE") - "ð" -> Ok("d") - "Ð" -> Ok("D") - "þ" -> Ok("th") - "Þ" -> Ok("Th") - "đ" | "ď" -> Ok("d") - "Đ" | "Ď" -> Ok("D") - "ġ" | "ğ" | "ĝ" | "ģ" -> Ok("g") - "Ġ" | "Ğ" | "Ĝ" | "Ģ" -> Ok("G") - "ĥ" | "ħ" -> Ok("h") - "Ĥ" | "Ħ" -> Ok("H") - "ĵ" -> Ok("j") - "Ĵ" -> Ok("J") - "ķ" -> Ok("k") - "Ķ" -> Ok("K") - "ĺ" | "ļ" | "ľ" | "ŀ" | "ł" -> Ok("l") - "Ĺ" | "Ļ" | "Ľ" | "Ŀ" | "Ł" -> Ok("L") - "ŕ" | "ŗ" | "ř" -> Ok("r") - "Ŕ" | "Ŗ" | "Ř" -> Ok("R") - "ś" | "ŝ" | "ş" | "š" | "ș" -> Ok("s") - "Ś" | "Ŝ" | "Ş" | "Š" | "Ș" -> Ok("S") - "ţ" | "ť" | "ŧ" | "ț" -> Ok("t") - "Ţ" | "Ť" | "Ŧ" | "Ț" -> Ok("T") - "ŵ" -> Ok("w") - "Ŵ" -> Ok("W") - "ź" | "ż" | "ž" -> Ok("z") - "Ź" | "Ż" | "Ž" -> Ok("Z") - _ -> Error(Nil) +fn locale_entries(locale: Locale) -> List(#(String, String)) { + case locale { + locale.German -> german + locale.Danish | locale.Norwegian -> danish_norwegian + locale.Default | locale.Swedish | locale.Turkish -> [] } } -/// Cyrillic characters (Russian alphabet plus common Ukrainian and -/// Belarusian letters), using a common romanization scheme. -pub fn lookup_cyrillic(grapheme: String) -> Result(String, Nil) { - case grapheme { - "а" -> Ok("a") - "б" -> Ok("b") - "в" -> Ok("v") - "г" -> Ok("g") - "д" -> Ok("d") - "е" -> Ok("e") - "ё" -> Ok("yo") - "ж" -> Ok("zh") - "з" -> Ok("z") - "и" -> Ok("i") - "й" -> Ok("y") - "к" -> Ok("k") - "л" -> Ok("l") - "м" -> Ok("m") - "н" -> Ok("n") - "о" -> Ok("o") - "п" -> Ok("p") - "р" -> Ok("r") - "с" -> Ok("s") - "т" -> Ok("t") - "у" -> Ok("u") - "ф" -> Ok("f") - "х" -> Ok("h") - "ц" -> Ok("ts") - "ч" -> Ok("ch") - "ш" -> Ok("sh") - "щ" -> Ok("shch") - "ъ" -> Ok("") - "ы" -> Ok("y") - "ь" -> Ok("") - "э" -> Ok("e") - "ю" -> Ok("yu") - "я" -> Ok("ya") - "є" -> Ok("ye") - "і" -> Ok("i") - "ї" -> Ok("yi") - "ґ" -> Ok("g") - "ў" -> Ok("u") - "А" -> Ok("A") - "Б" -> Ok("B") - "В" -> Ok("V") - "Г" -> Ok("G") - "Д" -> Ok("D") - "Е" -> Ok("E") - "Ё" -> Ok("Yo") - "Ж" -> Ok("Zh") - "З" -> Ok("Z") - "И" -> Ok("I") - "Й" -> Ok("Y") - "К" -> Ok("K") - "Л" -> Ok("L") - "М" -> Ok("M") - "Н" -> Ok("N") - "О" -> Ok("O") - "П" -> Ok("P") - "Р" -> Ok("R") - "С" -> Ok("S") - "Т" -> Ok("T") - "У" -> Ok("U") - "Ф" -> Ok("F") - "Х" -> Ok("H") - "Ц" -> Ok("Ts") - "Ч" -> Ok("Ch") - "Ш" -> Ok("Sh") - "Щ" -> Ok("Shch") - "Ъ" -> Ok("") - "Ы" -> Ok("Y") - "Ь" -> Ok("") - "Э" -> Ok("E") - "Ю" -> Ok("Yu") - "Я" -> Ok("Ya") - "Є" -> Ok("Ye") - "І" -> Ok("I") - "Ї" -> Ok("Yi") - "Ґ" -> Ok("G") - "Ў" -> Ok("U") - _ -> Error(Nil) - } -} +const german: List(#(String, String)) = [ + #("ä", "ae"), + #("ö", "oe"), + #("ü", "ue"), + #("Ä", "Ae"), + #("Ö", "Oe"), + #("Ü", "Ue"), +] -/// Greek characters (modern Greek romanization). -pub fn lookup_greek(grapheme: String) -> Result(String, Nil) { - case grapheme { - "α" | "ά" -> Ok("a") - "β" -> Ok("v") - "γ" -> Ok("g") - "δ" -> Ok("d") - "ε" | "έ" -> Ok("e") - "ζ" -> Ok("z") - "η" | "ή" -> Ok("i") - "θ" -> Ok("th") - "ι" | "ί" | "ϊ" | "ΐ" -> Ok("i") - "κ" -> Ok("k") - "λ" -> Ok("l") - "μ" -> Ok("m") - "ν" -> Ok("n") - "ξ" -> Ok("ks") - "ο" | "ό" -> Ok("o") - "π" -> Ok("p") - "ρ" -> Ok("r") - "σ" | "ς" -> Ok("s") - "τ" -> Ok("t") - "υ" | "ύ" | "ϋ" | "ΰ" -> Ok("y") - "φ" -> Ok("f") - "χ" -> Ok("x") - "ψ" -> Ok("ps") - "ω" | "ώ" -> Ok("o") - "Α" | "Ά" -> Ok("A") - "Β" -> Ok("V") - "Γ" -> Ok("G") - "Δ" -> Ok("D") - "Ε" | "Έ" -> Ok("E") - "Ζ" -> Ok("Z") - "Η" | "Ή" -> Ok("I") - "Θ" -> Ok("Th") - "Ι" | "Ί" -> Ok("I") - "Κ" -> Ok("K") - "Λ" -> Ok("L") - "Μ" -> Ok("M") - "Ν" -> Ok("N") - "Ξ" -> Ok("Ks") - "Ο" | "Ό" -> Ok("O") - "Π" -> Ok("P") - "Ρ" -> Ok("R") - "Σ" -> Ok("S") - "Τ" -> Ok("T") - "Υ" | "Ύ" -> Ok("Y") - "Φ" -> Ok("F") - "Χ" -> Ok("X") - "Ψ" -> Ok("Ps") - "Ω" | "Ώ" -> Ok("O") - _ -> Error(Nil) - } -} +const danish_norwegian: List(#(String, String)) = [ + #("æ", "ae"), + #("ø", "oe"), + #("å", "aa"), + #("Æ", "Ae"), + #("Ø", "Oe"), + #("Å", "Aa"), +] -/// Arabic and Persian characters, using a basic consonantal romanization -/// (vowel marks are dropped, as in other slugify libraries). -pub fn lookup_arabic(grapheme: String) -> Result(String, Nil) { - case grapheme { - "ا" | "أ" | "آ" | "ى" -> Ok("a") - "إ" -> Ok("i") - "ء" | "ئ" -> Ok("e") - "ؤ" -> Ok("w") - "ب" -> Ok("b") - "ت" -> Ok("t") - "ث" -> Ok("th") - "ج" -> Ok("j") - "ح" -> Ok("h") - "خ" -> Ok("kh") - "د" -> Ok("d") - "ذ" -> Ok("dh") - "ر" -> Ok("r") - "ز" -> Ok("z") - "س" -> Ok("s") - "ش" -> Ok("sh") - "ص" -> Ok("s") - "ض" -> Ok("d") - "ط" -> Ok("t") - "ظ" -> Ok("z") - "ع" -> Ok("a") - "غ" -> Ok("gh") - "ف" -> Ok("f") - "ق" -> Ok("q") - "ك" | "ک" -> Ok("k") - "ل" -> Ok("l") - "م" -> Ok("m") - "ن" -> Ok("n") - "ه" | "ة" -> Ok("h") - "و" -> Ok("w") - "ي" | "ی" -> Ok("y") - "پ" -> Ok("p") - "چ" -> Ok("ch") - "ژ" -> Ok("zh") - "گ" -> Ok("g") - "٠" | "۰" -> Ok("0") - "١" | "۱" -> Ok("1") - "٢" | "۲" -> Ok("2") - "٣" | "۳" -> Ok("3") - "٤" | "۴" -> Ok("4") - "٥" | "۵" -> Ok("5") - "٦" | "۶" -> Ok("6") - "٧" | "۷" -> Ok("7") - "٨" | "۸" -> Ok("8") - "٩" | "۹" -> Ok("9") - _ -> Error(Nil) - } -} +const latin: List(#(String, String)) = [ + #("à", "a"), + #("á", "a"), + #("â", "a"), + #("ã", "a"), + #("ä", "a"), + #("å", "a"), + #("ā", "a"), + #("ă", "a"), + #("ą", "a"), + #("À", "A"), + #("Á", "A"), + #("Â", "A"), + #("Ã", "A"), + #("Ä", "A"), + #("Å", "A"), + #("Ā", "A"), + #("Ă", "A"), + #("Ą", "A"), + #("è", "e"), + #("é", "e"), + #("ê", "e"), + #("ë", "e"), + #("ē", "e"), + #("ĕ", "e"), + #("ė", "e"), + #("ę", "e"), + #("ě", "e"), + #("È", "E"), + #("É", "E"), + #("Ê", "E"), + #("Ë", "E"), + #("Ē", "E"), + #("Ĕ", "E"), + #("Ė", "E"), + #("Ę", "E"), + #("Ě", "E"), + #("ì", "i"), + #("í", "i"), + #("î", "i"), + #("ï", "i"), + #("ī", "i"), + #("ĭ", "i"), + #("į", "i"), + #("ı", "i"), + #("Ì", "I"), + #("Í", "I"), + #("Î", "I"), + #("Ï", "I"), + #("Ī", "I"), + #("Ĭ", "I"), + #("Į", "I"), + #("İ", "I"), + #("ò", "o"), + #("ó", "o"), + #("ô", "o"), + #("õ", "o"), + #("ö", "o"), + #("ø", "o"), + #("ō", "o"), + #("ŏ", "o"), + #("ő", "o"), + #("Ò", "O"), + #("Ó", "O"), + #("Ô", "O"), + #("Õ", "O"), + #("Ö", "O"), + #("Ø", "O"), + #("Ō", "O"), + #("Ŏ", "O"), + #("Ő", "O"), + #("ù", "u"), + #("ú", "u"), + #("û", "u"), + #("ü", "u"), + #("ū", "u"), + #("ŭ", "u"), + #("ů", "u"), + #("ű", "u"), + #("ų", "u"), + #("Ù", "U"), + #("Ú", "U"), + #("Û", "U"), + #("Ü", "U"), + #("Ū", "U"), + #("Ŭ", "U"), + #("Ů", "U"), + #("Ű", "U"), + #("Ų", "U"), + #("ý", "y"), + #("ÿ", "y"), + #("ŷ", "y"), + #("Ý", "Y"), + #("Ÿ", "Y"), + #("Ŷ", "Y"), + #("ç", "c"), + #("ć", "c"), + #("ĉ", "c"), + #("ċ", "c"), + #("č", "c"), + #("Ç", "C"), + #("Ć", "C"), + #("Ĉ", "C"), + #("Ċ", "C"), + #("Č", "C"), + #("ñ", "n"), + #("ń", "n"), + #("ņ", "n"), + #("ň", "n"), + #("Ñ", "N"), + #("Ń", "N"), + #("Ņ", "N"), + #("Ň", "N"), + #("ß", "ss"), + #("æ", "ae"), + #("Æ", "AE"), + #("œ", "oe"), + #("Œ", "OE"), + #("ð", "d"), + #("Ð", "D"), + #("þ", "th"), + #("Þ", "Th"), + #("đ", "d"), + #("ď", "d"), + #("Đ", "D"), + #("Ď", "D"), + #("ġ", "g"), + #("ğ", "g"), + #("ĝ", "g"), + #("ģ", "g"), + #("Ġ", "G"), + #("Ğ", "G"), + #("Ĝ", "G"), + #("Ģ", "G"), + #("ĥ", "h"), + #("ħ", "h"), + #("Ĥ", "H"), + #("Ħ", "H"), + #("ĵ", "j"), + #("Ĵ", "J"), + #("ķ", "k"), + #("Ķ", "K"), + #("ĺ", "l"), + #("ļ", "l"), + #("ľ", "l"), + #("ŀ", "l"), + #("ł", "l"), + #("Ĺ", "L"), + #("Ļ", "L"), + #("Ľ", "L"), + #("Ŀ", "L"), + #("Ł", "L"), + #("ŕ", "r"), + #("ŗ", "r"), + #("ř", "r"), + #("Ŕ", "R"), + #("Ŗ", "R"), + #("Ř", "R"), + #("ś", "s"), + #("ŝ", "s"), + #("ş", "s"), + #("š", "s"), + #("ș", "s"), + #("Ś", "S"), + #("Ŝ", "S"), + #("Ş", "S"), + #("Š", "S"), + #("Ș", "S"), + #("ţ", "t"), + #("ť", "t"), + #("ŧ", "t"), + #("ț", "t"), + #("Ţ", "T"), + #("Ť", "T"), + #("Ŧ", "T"), + #("Ț", "T"), + #("ŵ", "w"), + #("Ŵ", "W"), + #("ź", "z"), + #("ż", "z"), + #("ž", "z"), + #("Ź", "Z"), + #("Ż", "Z"), + #("Ž", "Z"), +] -/// Hebrew characters, using a basic consonantal romanization -/// (niqqud vowel points are dropped). -pub fn lookup_hebrew(grapheme: String) -> Result(String, Nil) { - case grapheme { - "א" -> Ok("a") - "ב" -> Ok("b") - "ג" -> Ok("g") - "ד" -> Ok("d") - "ה" -> Ok("h") - "ו" -> Ok("v") - "ז" -> Ok("z") - "ח" -> Ok("h") - "ט" -> Ok("t") - "י" -> Ok("y") - "כ" | "ך" -> Ok("k") - "ל" -> Ok("l") - "מ" | "ם" -> Ok("m") - "נ" | "ן" -> Ok("n") - "ס" -> Ok("s") - "ע" -> Ok("a") - "פ" | "ף" -> Ok("p") - "צ" | "ץ" -> Ok("ts") - "ק" -> Ok("k") - "ר" -> Ok("r") - "ש" -> Ok("sh") - "ת" -> Ok("t") - _ -> Error(Nil) - } -} +const cyrillic: List(#(String, String)) = [ + #("а", "a"), + #("б", "b"), + #("в", "v"), + #("г", "g"), + #("д", "d"), + #("е", "e"), + #("ё", "yo"), + #("ж", "zh"), + #("з", "z"), + #("и", "i"), + #("й", "y"), + #("к", "k"), + #("л", "l"), + #("м", "m"), + #("н", "n"), + #("о", "o"), + #("п", "p"), + #("р", "r"), + #("с", "s"), + #("т", "t"), + #("у", "u"), + #("ф", "f"), + #("х", "h"), + #("ц", "ts"), + #("ч", "ch"), + #("ш", "sh"), + #("щ", "shch"), + #("ъ", ""), + #("ы", "y"), + #("ь", ""), + #("э", "e"), + #("ю", "yu"), + #("я", "ya"), + #("є", "ye"), + #("і", "i"), + #("ї", "yi"), + #("ґ", "g"), + #("ў", "u"), + #("А", "A"), + #("Б", "B"), + #("В", "V"), + #("Г", "G"), + #("Д", "D"), + #("Е", "E"), + #("Ё", "Yo"), + #("Ж", "Zh"), + #("З", "Z"), + #("И", "I"), + #("Й", "Y"), + #("К", "K"), + #("Л", "L"), + #("М", "M"), + #("Н", "N"), + #("О", "O"), + #("П", "P"), + #("Р", "R"), + #("С", "S"), + #("Т", "T"), + #("У", "U"), + #("Ф", "F"), + #("Х", "H"), + #("Ц", "Ts"), + #("Ч", "Ch"), + #("Ш", "Sh"), + #("Щ", "Shch"), + #("Ъ", ""), + #("Ы", "Y"), + #("Ь", ""), + #("Э", "E"), + #("Ю", "Yu"), + #("Я", "Ya"), + #("Є", "Ye"), + #("І", "I"), + #("Ї", "Yi"), + #("Ґ", "G"), + #("Ў", "U"), +] -/// Typographic punctuation commonly produced by word processors and CMSes. -/// -/// Apostrophe-like characters map to the empty string so contractions -/// stay joined ("don’t" becomes "dont"); everything else maps to a space, -/// which later becomes a separator. -pub fn lookup_punctuation(grapheme: String) -> Result(String, Nil) { - case grapheme { - "'" | "’" | "‘" | "‚" | "ʼ" | "ʻ" -> Ok("") - "“" | "”" | "„" | "‟" | "«" | "»" | "‹" | "›" | "′" | "″" -> Ok(" ") - "–" | "—" | "―" | "‒" | "‐" | "‑" -> Ok(" ") - "…" | "•" | "·" | "∙" | "¡" | "¿" | "§" | "¶" | "†" | "‡" -> Ok(" ") - // Unicode whitespace: no-break space, en/em and other typographic - // spaces, narrow no-break space, ideographic space, zero-width space - "\u{00A0}" - | "\u{2000}" - | "\u{2001}" - | "\u{2002}" - | "\u{2003}" - | "\u{2004}" - | "\u{2005}" - | "\u{2006}" - | "\u{2007}" - | "\u{2008}" - | "\u{2009}" - | "\u{200A}" - | "\u{200B}" - | "\u{202F}" - | "\u{3000}" -> Ok(" ") - _ -> Error(Nil) - } -} +const greek: List(#(String, String)) = [ + #("α", "a"), + #("ά", "a"), + #("β", "v"), + #("γ", "g"), + #("δ", "d"), + #("ε", "e"), + #("έ", "e"), + #("ζ", "z"), + #("η", "i"), + #("ή", "i"), + #("θ", "th"), + #("ι", "i"), + #("ί", "i"), + #("ϊ", "i"), + #("ΐ", "i"), + #("κ", "k"), + #("λ", "l"), + #("μ", "m"), + #("ν", "n"), + #("ξ", "ks"), + #("ο", "o"), + #("ό", "o"), + #("π", "p"), + #("ρ", "r"), + #("σ", "s"), + #("ς", "s"), + #("τ", "t"), + #("υ", "y"), + #("ύ", "y"), + #("ϋ", "y"), + #("ΰ", "y"), + #("φ", "f"), + #("χ", "x"), + #("ψ", "ps"), + #("ω", "o"), + #("ώ", "o"), + #("Α", "A"), + #("Ά", "A"), + #("Β", "V"), + #("Γ", "G"), + #("Δ", "D"), + #("Ε", "E"), + #("Έ", "E"), + #("Ζ", "Z"), + #("Η", "I"), + #("Ή", "I"), + #("Θ", "Th"), + #("Ι", "I"), + #("Ί", "I"), + #("Κ", "K"), + #("Λ", "L"), + #("Μ", "M"), + #("Ν", "N"), + #("Ξ", "Ks"), + #("Ο", "O"), + #("Ό", "O"), + #("Π", "P"), + #("Ρ", "R"), + #("Σ", "S"), + #("Τ", "T"), + #("Υ", "Y"), + #("Ύ", "Y"), + #("Φ", "F"), + #("Χ", "X"), + #("Ψ", "Ps"), + #("Ω", "O"), + #("Ώ", "O"), +] -/// Currency signs and common symbols, mapped to words. -pub fn lookup_symbol(grapheme: String) -> Result(String, Nil) { - case grapheme { - "&" -> Ok(" and ") - "@" -> Ok(" at ") - "%" -> Ok(" percent ") - "$" -> Ok(" dollar ") - "€" -> Ok(" euro ") - "£" -> Ok(" pound ") - "¥" -> Ok(" yen ") - "¢" -> Ok(" cent ") - "₹" -> Ok(" rupee ") - "₩" -> Ok(" won ") - "₽" -> Ok(" ruble ") - "₺" -> Ok(" lira ") - "₫" -> Ok(" dong ") - "฿" -> Ok(" baht ") - "©" -> Ok(" c ") - "®" -> Ok(" r ") - "™" -> Ok(" tm ") - "°" -> Ok(" degrees ") - "№" -> Ok(" no ") - "×" -> Ok(" x ") - "÷" -> Ok(" ") - "±" -> Ok(" plus ") - "µ" -> Ok("u") - _ -> Error(Nil) - } -} +const arabic: List(#(String, String)) = [ + #("ا", "a"), + #("أ", "a"), + #("آ", "a"), + #("ى", "a"), + #("إ", "i"), + #("ء", "e"), + #("ئ", "e"), + #("ؤ", "w"), + #("ب", "b"), + #("ت", "t"), + #("ث", "th"), + #("ج", "j"), + #("ح", "h"), + #("خ", "kh"), + #("د", "d"), + #("ذ", "dh"), + #("ر", "r"), + #("ز", "z"), + #("س", "s"), + #("ش", "sh"), + #("ص", "s"), + #("ض", "d"), + #("ط", "t"), + #("ظ", "z"), + #("ع", "a"), + #("غ", "gh"), + #("ف", "f"), + #("ق", "q"), + #("ك", "k"), + #("ک", "k"), + #("ل", "l"), + #("م", "m"), + #("ن", "n"), + #("ه", "h"), + #("ة", "h"), + #("و", "w"), + #("ي", "y"), + #("ی", "y"), + #("پ", "p"), + #("چ", "ch"), + #("ژ", "zh"), + #("گ", "g"), + #("٠", "0"), + #("۰", "0"), + #("١", "1"), + #("۱", "1"), + #("٢", "2"), + #("۲", "2"), + #("٣", "3"), + #("۳", "3"), + #("٤", "4"), + #("۴", "4"), + #("٥", "5"), + #("۵", "5"), + #("٦", "6"), + #("۶", "6"), + #("٧", "7"), + #("۷", "7"), + #("٨", "8"), + #("۸", "8"), + #("٩", "9"), + #("۹", "9"), +] + +const hebrew: List(#(String, String)) = [ + #("א", "a"), + #("ב", "b"), + #("ג", "g"), + #("ד", "d"), + #("ה", "h"), + #("ו", "v"), + #("ז", "z"), + #("ח", "h"), + #("ט", "t"), + #("י", "y"), + #("כ", "k"), + #("ך", "k"), + #("ל", "l"), + #("מ", "m"), + #("ם", "m"), + #("נ", "n"), + #("ן", "n"), + #("ס", "s"), + #("ע", "a"), + #("פ", "p"), + #("ף", "p"), + #("צ", "ts"), + #("ץ", "ts"), + #("ק", "k"), + #("ר", "r"), + #("ש", "sh"), + #("ת", "t"), +] + +const punctuation: List(#(String, String)) = [ + #("'", ""), + #("’", ""), + #("‘", ""), + #("‚", ""), + #("ʼ", ""), + #("ʻ", ""), + #("“", " "), + #("”", " "), + #("„", " "), + #("‟", " "), + #("«", " "), + #("»", " "), + #("‹", " "), + #("›", " "), + #("′", " "), + #("″", " "), + #("–", " "), + #("—", " "), + #("―", " "), + #("‒", " "), + #("‐", " "), + #("‑", " "), + #("…", " "), + #("•", " "), + #("·", " "), + #("∙", " "), + #("¡", " "), + #("¿", " "), + #("§", " "), + #("¶", " "), + #("†", " "), + #("‡", " "), + #("\u{00A0}", " "), + #("\u{2000}", " "), + #("\u{2001}", " "), + #("\u{2002}", " "), + #("\u{2003}", " "), + #("\u{2004}", " "), + #("\u{2005}", " "), + #("\u{2006}", " "), + #("\u{2007}", " "), + #("\u{2008}", " "), + #("\u{2009}", " "), + #("\u{200A}", " "), + #("\u{200B}", " "), + #("\u{202F}", " "), + #("\u{3000}", " "), +] + +const symbol: List(#(String, String)) = [ + #("&", " and "), + #("@", " at "), + #("%", " percent "), + #("$", " dollar "), + #("€", " euro "), + #("£", " pound "), + #("¥", " yen "), + #("¢", " cent "), + #("₹", " rupee "), + #("₩", " won "), + #("₽", " ruble "), + #("₺", " lira "), + #("₫", " dong "), + #("฿", " baht "), + #("©", " c "), + #("®", " r "), + #("™", " tm "), + #("°", " degrees "), + #("№", " no "), + #("×", " x "), + #("÷", " "), + #("±", " plus "), + #("µ", "u"), +] diff --git a/src/glugify/internal/entities.gleam b/src/glugify/internal/entities.gleam deleted file mode 100644 index 6678e1b..0000000 --- a/src/glugify/internal/entities.gleam +++ /dev/null @@ -1,104 +0,0 @@ -import gleam/int -import gleam/list -import gleam/string - -/// Decodes HTML entities in text: named entities with slug-relevant -/// meanings (`&`, ` `, `—`, ...), decimal numeric -/// references (`&`) and hexadecimal references (`&`). -/// Sequences that are not valid entities are left untouched. -pub fn decode(text: String) -> String { - text - |> string.to_graphemes - |> decode_graphemes([]) - |> string.join("") -} - -fn decode_graphemes( - graphemes: List(String), - acc: List(String), -) -> List(String) { - case graphemes { - [] -> list.reverse(acc) - ["&", ..rest] -> - case take_entity(rest, [], 0) { - Ok(#(name, remaining)) -> - case decode_entity(name) { - Ok(replacement) -> decode_graphemes(remaining, [replacement, ..acc]) - Error(Nil) -> decode_graphemes(rest, ["&", ..acc]) - } - Error(Nil) -> decode_graphemes(rest, ["&", ..acc]) - } - [grapheme, ..rest] -> decode_graphemes(rest, [grapheme, ..acc]) - } -} - -/// Collects the entity name between "&" and ";", up to a sane length. -fn take_entity( - graphemes: List(String), - acc: List(String), - length: Int, -) -> Result(#(String, List(String)), Nil) { - case graphemes { - [";", ..rest] -> - case acc { - [] -> Error(Nil) - _ -> Ok(#(string.join(list.reverse(acc), ""), rest)) - } - [grapheme, ..rest] -> - case length > 10 { - True -> Error(Nil) - False -> take_entity(rest, [grapheme, ..acc], length + 1) - } - [] -> Error(Nil) - } -} - -fn decode_entity(name: String) -> Result(String, Nil) { - case name { - "amp" -> Ok("&") - "lt" -> Ok("<") - "gt" -> Ok(">") - "quot" -> Ok("\"") - "apos" -> Ok("'") - "nbsp" -> Ok("\u{00A0}") - "ndash" -> Ok("–") - "mdash" -> Ok("—") - "hellip" -> Ok("…") - "lsquo" -> Ok("‘") - "rsquo" -> Ok("’") - "ldquo" -> Ok("“") - "rdquo" -> Ok("”") - "copy" -> Ok("©") - "reg" -> Ok("®") - "trade" -> Ok("™") - "deg" -> Ok("°") - "euro" -> Ok("€") - "pound" -> Ok("£") - "yen" -> Ok("¥") - "cent" -> Ok("¢") - _ -> decode_numeric_entity(name) - } -} - -fn decode_numeric_entity(name: String) -> Result(String, Nil) { - case name { - "#x" <> hex | "#X" <> hex -> - case int.base_parse(hex, 16) { - Ok(code) -> codepoint_to_string(code) - Error(Nil) -> Error(Nil) - } - "#" <> decimal -> - case int.parse(decimal) { - Ok(code) -> codepoint_to_string(code) - Error(Nil) -> Error(Nil) - } - _ -> Error(Nil) - } -} - -fn codepoint_to_string(code: Int) -> Result(String, Nil) { - case string.utf_codepoint(code) { - Ok(codepoint) -> Ok(string.from_utf_codepoints([codepoint])) - Error(Nil) -> Error(Nil) - } -} diff --git a/src/glugify/internal/pipeline.gleam b/src/glugify/internal/pipeline.gleam new file mode 100644 index 0000000..97efda4 --- /dev/null +++ b/src/glugify/internal/pipeline.gleam @@ -0,0 +1,729 @@ +import gleam/int +import gleam/list +import gleam/option +import gleam/result +import gleam/string +import glugify/config.{type Config} +import glugify/errors.{type SlugifyError} +import glugify/unicode + +pub fn run(text: String, config: Config) -> Result(String, SlugifyError) { + use validated <- result.try(validate_input(text)) + use normalized <- result.try(normalize(validated, config)) + use transliterated <- result.try(transliterate(normalized, config)) + use separated <- result.try(separate(transliterated, config)) + use finished <- result.try(finish(separated, config)) + Ok(apply_preserve_options(finished, text, config)) +} + +fn normalize(text: String, config: Config) -> Result(String, SlugifyError) { + use decoded <- result.try(decode_entities(text, config)) + use spaced <- result.try(normalize_whitespace(decoded)) + use decamelized <- result.try(split_camel_case(spaced, config)) + apply_custom_replacements(decamelized, config.custom_replacements) +} + +fn decode_entities( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.decode_html_entities { + True -> Ok(decode_html(text)) + False -> Ok(text) + } +} + +fn split_camel_case( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.decamelize { + True -> decamelize(text) + False -> Ok(text) + } +} + +fn transliterate(text: String, config: Config) -> Result(String, SlugifyError) { + use converted <- result.try(to_permitted_charset(text, config)) + use spaced <- result.try(normalize_whitespace(converted)) + Ok(apply_lowercase(spaced, config)) +} + +fn to_permitted_charset( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.transliterate { + True -> unicode.transliterate_text_with(text, config.locale, config.ignore) + False -> + unicode.validate_ascii_or_unicode( + text, + config.allow_unicode, + config.ignore, + ) + } +} + +fn apply_lowercase(text: String, config: Config) -> String { + case config.lowercase { + True -> string.lowercase(text) + False -> text + } +} + +fn separate(text: String, config: Config) -> Result(String, SlugifyError) { + use pre_filtered <- result.try(filter_stop_words_while_spaces_remain( + text, + config, + )) + use separated <- result.try(apply_separators(pre_filtered, config)) + collapse_separators(separated, config) +} + +fn filter_stop_words_while_spaces_remain( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.separator { + "" -> filter_stop_words(text, config.stop_words, " ") + _ -> Ok(text) + } +} + +fn finish(text: String, config: Config) -> Result(String, SlugifyError) { + use filtered <- result.try(filter_stop_words_on_separator(text, config)) + use trimmed <- result.try(trim_separators(filtered, config)) + truncate(trimmed, config) +} + +fn filter_stop_words_on_separator( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.separator { + "" -> Ok(text) + _ -> filter_stop_words(text, config.stop_words, config.separator) + } +} + +fn truncate(text: String, config: Config) -> Result(String, SlugifyError) { + case config.max_length { + option.Some(length) -> + truncate_slug(text, length, config.word_boundary, config.separator) + option.None -> Ok(text) + } +} + +fn apply_preserve_options( + slug: String, + input: String, + config: Config, +) -> String { + slug + |> keep_leading_underscore(input, config) + |> keep_trailing_separator(input, config) +} + +fn keep_leading_underscore( + slug: String, + input: String, + config: Config, +) -> String { + case underscore_was_dropped(slug, input, config) { + True -> "_" <> slug + False -> slug + } +} + +fn underscore_was_dropped(slug: String, input: String, config: Config) -> Bool { + config.preserve_leading_underscore + && string.starts_with(input, "_") + && !string.starts_with(slug, "_") +} + +fn keep_trailing_separator( + slug: String, + input: String, + config: Config, +) -> String { + case trailing_dash_was_dropped(slug, input, config) { + True -> slug <> config.separator + False -> slug + } +} + +fn trailing_dash_was_dropped( + slug: String, + input: String, + config: Config, +) -> Bool { + config.preserve_trailing_dash + && string.ends_with(input, "-") + && config.separator != "" + && !string.ends_with(slug, config.separator) +} + +fn validate_input(text: String) -> Result(String, SlugifyError) { + case string.trim(text) { + "" -> Error(errors.EmptyInput) + trimmed -> Ok(trimmed) + } +} + +const longest_entity_name = 10 + +const named_entities: List(#(String, String)) = [ + #("amp", "&"), + #("lt", "<"), + #("gt", ">"), + #("quot", "\""), + #("apos", "'"), + #("nbsp", "\u{00A0}"), + #("ndash", "–"), + #("mdash", "—"), + #("hellip", "…"), + #("lsquo", "‘"), + #("rsquo", "’"), + #("ldquo", "“"), + #("rdquo", "”"), + #("copy", "©"), + #("reg", "®"), + #("trade", "™"), + #("deg", "°"), + #("euro", "€"), + #("pound", "£"), + #("yen", "¥"), + #("cent", "¢"), +] + +fn decode_html(text: String) -> String { + text + |> string.to_graphemes + |> decode_graphemes([]) + |> string.join("") +} + +fn decode_graphemes( + graphemes: List(String), + acc: List(String), +) -> List(String) { + case graphemes { + [] -> list.reverse(acc) + ["&", ..rest] -> { + let #(remaining, next) = decode_after_ampersand(rest, acc) + decode_graphemes(remaining, next) + } + [grapheme, ..rest] -> decode_graphemes(rest, [grapheme, ..acc]) + } +} + +fn decode_after_ampersand( + rest: List(String), + acc: List(String), +) -> #(List(String), List(String)) { + case take_entity(rest, [], 0) { + Ok(#(name, remaining)) -> replace_entity(name, remaining, rest, acc) + Error(Nil) -> #(rest, ["&", ..acc]) + } +} + +fn replace_entity( + name: String, + remaining: List(String), + rest: List(String), + acc: List(String), +) -> #(List(String), List(String)) { + case decode_entity(name) { + Ok(replacement) -> #(remaining, [replacement, ..acc]) + Error(Nil) -> #(rest, ["&", ..acc]) + } +} + +fn take_entity( + graphemes: List(String), + acc: List(String), + length: Int, +) -> Result(#(String, List(String)), Nil) { + case graphemes { + [";", ..rest] -> finish_entity(acc, rest) + [grapheme, ..rest] -> extend_entity(grapheme, rest, acc, length) + [] -> Error(Nil) + } +} + +fn finish_entity( + acc: List(String), + rest: List(String), +) -> Result(#(String, List(String)), Nil) { + case acc { + [] -> Error(Nil) + _ -> Ok(#(string.join(list.reverse(acc), ""), rest)) + } +} + +fn extend_entity( + grapheme: String, + rest: List(String), + acc: List(String), + length: Int, +) -> Result(#(String, List(String)), Nil) { + case length > longest_entity_name { + True -> Error(Nil) + False -> take_entity(rest, [grapheme, ..acc], length + 1) + } +} + +fn decode_entity(name: String) -> Result(String, Nil) { + case list.key_find(named_entities, name) { + Ok(replacement) -> Ok(replacement) + Error(Nil) -> decode_numeric_entity(name) + } +} + +fn decode_numeric_entity(name: String) -> Result(String, Nil) { + case name { + "#x" <> hex | "#X" <> hex -> decode_hexadecimal(hex) + "#" <> decimal -> decode_decimal(decimal) + _ -> Error(Nil) + } +} + +fn decode_hexadecimal(hex: String) -> Result(String, Nil) { + int.base_parse(hex, 16) + |> result.try(codepoint_to_string) +} + +fn decode_decimal(decimal: String) -> Result(String, Nil) { + int.parse(decimal) + |> result.try(codepoint_to_string) +} + +fn codepoint_to_string(code: Int) -> Result(String, Nil) { + case string.utf_codepoint(code) { + Ok(codepoint) -> Ok(string.from_utf_codepoints([codepoint])) + Error(Nil) -> Error(Nil) + } +} + +const digits = #(48, 57) + +const uppercase_letters = #(65, 90) + +const lowercase_letters = #(97, 122) + +const alphanumeric_ranges: List(#(Int, Int)) = [ + digits, uppercase_letters, lowercase_letters, +] + +const lower_or_digit_ranges: List(#(Int, Int)) = [lowercase_letters, digits] + +const highest_ascii_codepoint = 127 + +fn normalize_whitespace(text: String) -> Result(String, SlugifyError) { + text + |> string.to_graphemes + |> list.fold([], collapse_whitespace) + |> list.reverse + |> string.join("") + |> Ok +} + +fn collapse_whitespace(acc: List(String), char: String) -> List(String) { + case is_whitespace(char), acc { + True, [" ", ..] -> acc + True, _ -> [" ", ..acc] + False, _ -> [char, ..acc] + } +} + +fn is_whitespace(char: String) -> Bool { + case char { + " " | "\t" | "\n" | "\r" -> True + _ -> False + } +} + +fn apply_separators( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.separator { + "" -> + text + |> string.to_graphemes + |> list.filter(fn(char) { is_slug_char(char, config) }) + |> string.join("") + |> Ok + _ -> + text + |> string.to_graphemes + |> separate_graphemes(config) + |> string.join("") + |> Ok + } +} + +fn separate_graphemes(graphemes: List(String), config: Config) -> List(String) { + let #(acc, _) = + list.fold(graphemes, #([], False), fn(state, char) { + separate_step(state, char, config) + }) + list.reverse(acc) +} + +fn separate_step( + state: #(List(String), Bool), + char: String, + config: Config, +) -> #(List(String), Bool) { + let #(acc, last_was_separator) = state + case is_slug_char(char, config) { + True -> #([char, ..acc], False) + False -> #( + skip_or_separate(acc, last_was_separator, config.separator), + True, + ) + } +} + +fn skip_or_separate( + acc: List(String), + last_was_separator: Bool, + separator: String, +) -> List(String) { + case last_was_separator || list.is_empty(acc) { + True -> acc + False -> prepend_separator(acc, separator) + } +} + +fn prepend_separator(acc: List(String), separator: String) -> List(String) { + list.append(list.reverse(string.to_graphemes(separator)), acc) +} + +fn is_slug_char(char: String, config: Config) -> Bool { + is_alphanumeric_or_unicode(char, config.allow_unicode) + || is_ignored(char, config) +} + +fn is_ignored(char: String, config: Config) -> Bool { + list.contains(config.ignore, char) || matches_lowercased_entry(char, config) +} + +fn matches_lowercased_entry(char: String, config: Config) -> Bool { + config.lowercase + && list.any(config.ignore, fn(entry) { string.lowercase(entry) == char }) +} + +fn ends_with_separator(acc: List(String), separator: String) -> Bool { + matches_prefix(acc, list.reverse(string.to_graphemes(separator))) +} + +fn matches_prefix(haystack: List(String), needle: List(String)) -> Bool { + case needle, haystack { + [], _ -> True + [n, ..ns], [h, ..hs] if h == n -> matches_prefix(hs, ns) + _, _ -> False + } +} + +fn within(code: Int, range: #(Int, Int)) -> Bool { + code >= range.0 && code <= range.1 +} + +fn single_codepoint(char: String) -> Result(Int, Nil) { + case string.to_utf_codepoints(char) { + [codepoint] -> Ok(string.utf_codepoint_to_int(codepoint)) + _ -> Error(Nil) + } +} + +fn code_in_ranges(char: String, ranges: List(#(Int, Int))) -> Bool { + case single_codepoint(char) { + Ok(code) -> list.any(ranges, fn(range) { within(code, range) }) + Error(Nil) -> False + } +} + +fn is_alphanumeric(char: String) -> Bool { + code_in_ranges(char, alphanumeric_ranges) +} + +fn is_alphanumeric_or_unicode(char: String, allow_unicode: Bool) -> Bool { + case allow_unicode { + True -> is_alphanumeric(char) || is_unicode_char(char) + False -> is_alphanumeric(char) + } +} + +fn is_unicode_char(char: String) -> Bool { + list.any(string.to_utf_codepoints(char), is_above_ascii) +} + +fn is_above_ascii(codepoint: UtfCodepoint) -> Bool { + string.utf_codepoint_to_int(codepoint) > highest_ascii_codepoint +} + +fn collapse_separators( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.separator { + "" -> Ok(text) + separator -> + text + |> string.to_graphemes + |> collapse_repeated_separators(separator, []) + |> string.join("") + |> Ok + } +} + +fn collapse_repeated_separators( + graphemes: List(String), + separator: String, + acc: List(String), +) -> List(String) { + case graphemes { + [] -> list.reverse(acc) + [char, ..rest] -> { + let #(remaining, next) = + collapse_step(graphemes, char, rest, separator, acc) + collapse_repeated_separators(remaining, separator, next) + } + } +} + +fn collapse_step( + graphemes: List(String), + char: String, + rest: List(String), + separator: String, + acc: List(String), +) -> #(List(String), List(String)) { + case + starts_with_separator(graphemes, separator), + ends_with_separator(acc, separator) + { + True, True -> #(drop_separator_prefix(graphemes, separator), acc) + True, False -> #( + drop_separator_prefix(graphemes, separator), + prepend_separator(acc, separator), + ) + False, _ -> #(rest, [char, ..acc]) + } +} + +fn starts_with_separator(graphemes: List(String), separator: String) -> Bool { + matches_prefix(graphemes, string.to_graphemes(separator)) +} + +fn drop_separator_prefix( + graphemes: List(String), + separator: String, +) -> List(String) { + list.drop(graphemes, string.length(separator)) +} + +fn trim_separators( + text: String, + config: Config, +) -> Result(String, SlugifyError) { + case config.trim { + True -> + text + |> string.trim_start + |> string.trim_end + |> trim_separator_start(config.separator) + |> trim_separator_end(config.separator) + |> Ok + False -> Ok(text) + } +} + +fn trim_separator_start(text: String, separator: String) -> String { + let cleaned = drop_leading_separator(text, separator) + case cleaned == text { + True -> text + False -> trim_separator_start(cleaned, separator) + } +} + +fn trim_separator_end(text: String, separator: String) -> String { + let cleaned = drop_trailing_separator(text, separator) + case cleaned == text { + True -> text + False -> trim_separator_end(cleaned, separator) + } +} + +fn drop_leading_separator(text: String, separator: String) -> String { + case separator != "" && string.starts_with(text, separator) { + True -> string.drop_start(text, string.length(separator)) + False -> text + } +} + +fn truncate_slug( + text: String, + max_length: Int, + word_boundary: Bool, + separator: String, +) -> Result(String, SlugifyError) { + case string.length(text) <= max_length, word_boundary { + True, _ -> Ok(text) + False, True -> truncate_at_word_boundary(text, max_length, separator) + False, False -> truncate_without_word_boundary(text, max_length, separator) + } +} + +fn truncate_without_word_boundary( + text: String, + max_length: Int, + separator: String, +) -> Result(String, SlugifyError) { + text + |> string.slice(0, max_length) + |> drop_trailing_separator(separator) + |> Ok +} + +fn truncate_at_word_boundary( + text: String, + max_length: Int, + separator: String, +) -> Result(String, SlugifyError) { + let truncated = string.slice(text, 0, max_length) + let without_trailing = drop_trailing_separator(truncated, separator) + case without_trailing == truncated { + False -> Ok(without_trailing) + True -> Ok(cut_at_last_separator(truncated, separator)) + } +} + +fn drop_trailing_separator(text: String, separator: String) -> String { + case separator != "" && string.ends_with(text, separator) { + True -> string.drop_end(text, string.length(separator)) + False -> text + } +} + +fn cut_at_last_separator(text: String, separator: String) -> String { + case find_last_separator(text, separator) { + Ok(index) -> string.slice(text, 0, index) + Error(Nil) -> text + } +} + +fn find_last_separator(text: String, separator: String) -> Result(Int, Nil) { + find_last_separator_by_string( + text, + separator, + string.length(text) - string.length(separator), + ) +} + +fn find_last_separator_by_string( + text: String, + separator: String, + start_index: Int, +) -> Result(Int, Nil) { + case + start_index < 0, + string.slice(text, start_index, string.length(separator)) == separator + { + True, _ -> Error(Nil) + False, True -> Ok(start_index) + False, False -> + find_last_separator_by_string(text, separator, start_index - 1) + } +} + +fn decamelize(text: String) -> Result(String, SlugifyError) { + text + |> string.to_graphemes + |> decamelize_graphemes([]) + |> string.join("") + |> Ok +} + +fn decamelize_graphemes( + graphemes: List(String), + acc: List(String), +) -> List(String) { + case graphemes { + [a, b, ..rest] -> + decamelize_graphemes( + [b, ..rest], + push_grapheme(a, acc, starts_new_word(a, b, rest)), + ) + _ -> list.append(list.reverse(acc), graphemes) + } +} + +fn push_grapheme( + grapheme: String, + acc: List(String), + new_word: Bool, +) -> List(String) { + case new_word { + True -> [" ", grapheme, ..acc] + False -> [grapheme, ..acc] + } +} + +fn starts_new_word(a: String, b: String, rest: List(String)) -> Bool { + case camel_boundary(a, b), rest { + True, _ -> True + False, [c, ..] -> acronym_boundary(a, b, c) + False, [] -> False + } +} + +fn camel_boundary(a: String, b: String) -> Bool { + is_lower_or_digit(a) && is_upper(b) +} + +fn acronym_boundary(a: String, b: String, c: String) -> Bool { + is_upper(a) && is_upper(b) && is_lower_or_digit(c) +} + +fn is_upper(char: String) -> Bool { + code_in_ranges(char, [uppercase_letters]) +} + +fn is_lower_or_digit(char: String) -> Bool { + code_in_ranges(char, lower_or_digit_ranges) +} + +fn apply_custom_replacements( + text: String, + replacements: List(#(String, String)), +) -> Result(String, SlugifyError) { + list.fold(replacements, text, fn(acc, replacement) { + string.replace(acc, replacement.0, replacement.1) + }) + |> Ok +} + +fn filter_stop_words( + text: String, + stop_words: List(String), + separator: String, +) -> Result(String, SlugifyError) { + case list.is_empty(stop_words) || separator == "" { + True -> Ok(text) + False -> Ok(drop_stop_words(text, stop_words, separator)) + } +} + +fn drop_stop_words( + text: String, + stop_words: List(String), + separator: String, +) -> String { + let lowered = list.map(stop_words, string.lowercase) + string.split(text, separator) + |> list.filter(fn(word) { !list.contains(lowered, string.lowercase(word)) }) + |> string.join(separator) +} diff --git a/src/glugify/internal/processors.gleam b/src/glugify/internal/processors.gleam deleted file mode 100644 index 541dea6..0000000 --- a/src/glugify/internal/processors.gleam +++ /dev/null @@ -1,557 +0,0 @@ -import gleam/list -import gleam/string -import glugify/config.{type Config} -import glugify/errors.{type SlugifyError} - -pub fn normalize_whitespace(text: String) -> Result(String, SlugifyError) { - text - |> string.to_graphemes - |> normalize_whitespace_graphemes([]) - |> string.join("") - |> Ok -} - -fn normalize_whitespace_graphemes( - graphemes: List(String), - acc: List(String), -) -> List(String) { - case graphemes { - [] -> list.reverse(acc) - [char, ..rest] -> { - case is_whitespace(char) { - True -> { - case acc { - [" ", ..] -> normalize_whitespace_graphemes(rest, acc) - _ -> normalize_whitespace_graphemes(rest, [" ", ..acc]) - } - } - False -> normalize_whitespace_graphemes(rest, [char, ..acc]) - } - } - } -} - -fn is_whitespace(char: String) -> Bool { - case char { - " " | "\t" | "\n" | "\r" -> True - _ -> False - } -} - -pub fn apply_separators( - text: String, - config: Config, -) -> Result(String, SlugifyError) { - case config.separator { - "" -> { - text - |> string.to_graphemes - |> list.filter(fn(char) { is_slug_char(char, config) }) - |> string.join("") - |> Ok - } - _ -> { - text - |> string.to_graphemes - |> apply_separators_simple(config, [], False) - |> string.join("") - |> Ok - } - } -} - -fn apply_separators_simple( - graphemes: List(String), - config: Config, - acc: List(String), - last_was_separator: Bool, -) -> List(String) { - case graphemes { - [] -> list.reverse(acc) - [char, ..rest] -> { - case is_slug_char(char, config) { - True -> apply_separators_simple(rest, config, [char, ..acc], False) - False -> { - case last_was_separator || list.is_empty(acc) { - True -> apply_separators_simple(rest, config, acc, True) - False -> { - let separator_chars = - string.to_graphemes(config.separator) |> list.reverse - apply_separators_simple( - rest, - config, - list.append(separator_chars, acc), - True, - ) - } - } - } - } - } - } -} - -/// A grapheme that belongs in the slug body: alphanumeric, permitted -/// unicode, or explicitly ignored by configuration. -fn is_slug_char(char: String, config: Config) -> Bool { - is_alphanumeric_or_unicode(char, config.allow_unicode) - || is_ignored(char, config) -} - -/// Ignored graphemes are matched by their lowercased form too when the -/// config lowercases the slug, since these checks run after the lowercase -/// stage: with `ignore: ["Ü"]` the text "Ü" has already become "ü". -fn is_ignored(char: String, config: Config) -> Bool { - list.contains(config.ignore, char) - || { - config.lowercase - && list.any(config.ignore, fn(entry) { string.lowercase(entry) == char }) - } -} - -fn ends_with_separator(acc: List(String), separator: String) -> Bool { - case separator { - "" -> False - _ -> { - let separator_chars = string.to_graphemes(separator) |> list.reverse - ends_with_separator_helper(acc, separator_chars) - } - } -} - -fn ends_with_separator_helper( - acc: List(String), - separator_chars: List(String), -) -> Bool { - case separator_chars { - [] -> True - [sep_char, ..rest_sep] -> { - case acc { - [acc_char, ..rest_acc] if acc_char == sep_char -> - ends_with_separator_helper(rest_acc, rest_sep) - _ -> False - } - } - } -} - -fn is_alphanumeric(char: String) -> Bool { - case string.to_utf_codepoints(char) { - [codepoint] -> { - let code = string.utf_codepoint_to_int(codepoint) - { code >= 48 && code <= 57 } - // 0-9 - || { code >= 65 && code <= 90 } - // A-Z - || { code >= 97 && code <= 122 } - // a-z - } - _ -> False - } -} - -fn is_alphanumeric_or_unicode(char: String, allow_unicode: Bool) -> Bool { - case allow_unicode { - True -> is_alphanumeric(char) || is_unicode_char(char) - False -> is_alphanumeric(char) - } -} - -fn is_unicode_char(char: String) -> Bool { - // A grapheme may span several codepoints (decomposed accents, emoji - // with modifiers or joiners); treat it as unicode when any codepoint - // is outside ASCII. - case string.to_utf_codepoints(char) { - [] -> False - codepoints -> - list.any(codepoints, fn(codepoint) { - string.utf_codepoint_to_int(codepoint) > 127 - }) - } -} - -fn is_char_in_separator(char: String, separator: String) -> Bool { - case separator { - "" -> False - _ -> string.contains(separator, char) - } -} - -pub fn remove_invalid_chars( - text: String, - config: Config, -) -> Result(String, SlugifyError) { - text - |> string.to_graphemes - |> filter_valid_chars_with_unicode(config, []) - |> string.join("") - |> Ok -} - -fn filter_valid_chars_with_unicode( - graphemes: List(String), - config: Config, - acc: List(String), -) -> List(String) { - case graphemes { - [] -> list.reverse(acc) - [char, ..rest] -> { - case - is_slug_char(char, config) - || is_char_in_separator(char, config.separator) - { - True -> filter_valid_chars_with_unicode(rest, config, [char, ..acc]) - False -> filter_valid_chars_with_unicode(rest, config, acc) - } - } - } -} - -pub fn collapse_separators( - text: String, - config: Config, -) -> Result(String, SlugifyError) { - text - |> string.to_graphemes - |> collapse_separators_graphemes(config.separator, []) - |> string.join("") - |> Ok -} - -fn collapse_separators_graphemes( - graphemes: List(String), - separator: String, - acc: List(String), -) -> List(String) { - case separator { - "" -> list.reverse(list.append(list.reverse(graphemes), acc)) - _ -> collapse_separators_helper(graphemes, separator, acc) - } -} - -fn collapse_separators_helper( - graphemes: List(String), - separator: String, - acc: List(String), -) -> List(String) { - case graphemes { - [] -> list.reverse(acc) - _ -> { - case starts_with_separator(graphemes, separator) { - True -> { - case ends_with_separator(acc, separator) { - True -> { - let remaining = drop_separator_prefix(graphemes, separator) - collapse_separators_helper(remaining, separator, acc) - } - False -> { - let separator_chars = string.to_graphemes(separator) - let remaining = drop_separator_prefix(graphemes, separator) - collapse_separators_helper( - remaining, - separator, - list.append(list.reverse(separator_chars), acc), - ) - } - } - } - False -> { - case graphemes { - [char, ..rest] -> - collapse_separators_helper(rest, separator, [char, ..acc]) - [] -> list.reverse(acc) - } - } - } - } - } -} - -fn starts_with_separator(graphemes: List(String), separator: String) -> Bool { - let separator_chars = string.to_graphemes(separator) - starts_with_separator_helper(graphemes, separator_chars) -} - -fn starts_with_separator_helper( - graphemes: List(String), - separator_chars: List(String), -) -> Bool { - case separator_chars { - [] -> True - [sep_char, ..rest_sep] -> { - case graphemes { - [grapheme, ..rest_graphemes] if grapheme == sep_char -> - starts_with_separator_helper(rest_graphemes, rest_sep) - _ -> False - } - } - } -} - -fn drop_separator_prefix( - graphemes: List(String), - separator: String, -) -> List(String) { - let separator_chars = string.to_graphemes(separator) - drop_separator_prefix_helper(graphemes, separator_chars) -} - -fn drop_separator_prefix_helper( - graphemes: List(String), - separator_chars: List(String), -) -> List(String) { - case separator_chars { - [] -> graphemes - [_, ..rest_sep] -> { - case graphemes { - [_, ..rest_graphemes] -> - drop_separator_prefix_helper(rest_graphemes, rest_sep) - [] -> [] - } - } - } -} - -pub fn trim_separators( - text: String, - config: Config, -) -> Result(String, SlugifyError) { - case config.trim { - True -> { - text - |> string.trim_start - |> string.trim_end - |> trim_separator_ends(config.separator) - |> Ok - } - False -> Ok(text) - } -} - -fn trim_separator_ends(text: String, separator: String) -> String { - text - |> trim_separator_start(separator) - |> trim_separator_end(separator) -} - -fn trim_separator_start(text: String, separator: String) -> String { - case separator { - "" -> text - _ -> - case string.starts_with(text, separator) { - True -> - trim_separator_start( - string.drop_start(text, string.length(separator)), - separator, - ) - False -> text - } - } -} - -fn trim_separator_end(text: String, separator: String) -> String { - case separator { - "" -> text - _ -> - case string.ends_with(text, separator) { - True -> - trim_separator_end( - string.drop_end(text, string.length(separator)), - separator, - ) - False -> text - } - } -} - -pub fn truncate_slug( - text: String, - max_length: Int, - word_boundary: Bool, - separator: String, -) -> Result(String, SlugifyError) { - case string.length(text) <= max_length { - True -> Ok(text) - False -> { - case word_boundary { - True -> truncate_at_word_boundary(text, max_length, separator) - False -> truncate_without_word_boundary(text, max_length, separator) - } - } - } -} - -fn truncate_without_word_boundary( - text: String, - max_length: Int, - separator: String, -) -> Result(String, SlugifyError) { - let truncated = string.slice(text, 0, max_length) - case separator { - "" -> Ok(truncated) - _ -> { - case string.ends_with(truncated, separator) { - True -> Ok(string.drop_end(truncated, string.length(separator))) - False -> Ok(truncated) - } - } - } -} - -fn truncate_at_word_boundary( - text: String, - max_length: Int, - separator: String, -) -> Result(String, SlugifyError) { - let truncated = string.slice(text, 0, max_length) - case separator { - "" -> Ok(truncated) - _ -> { - case string.ends_with(truncated, separator) { - True -> Ok(string.drop_end(truncated, string.length(separator))) - False -> { - case find_last_separator(truncated, separator) { - Ok(index) -> Ok(string.slice(truncated, 0, index)) - Error(_) -> Ok(truncated) - } - } - } - } - } -} - -fn find_last_separator(text: String, separator: String) -> Result(Int, Nil) { - find_last_separator_by_string( - text, - separator, - string.length(text) - string.length(separator), - ) -} - -fn find_last_separator_by_string( - text: String, - separator: String, - start_index: Int, -) -> Result(Int, Nil) { - case start_index < 0 { - True -> Error(Nil) - False -> { - let substr = string.slice(text, start_index, string.length(separator)) - case substr == separator { - True -> Ok(start_index) - False -> find_last_separator_by_string(text, separator, start_index - 1) - } - } - } -} - -/// Inserts spaces at camelCase word boundaries so each word slugifies -/// separately: "fooBar" -> "foo Bar", "HTMLParser" -> "HTML Parser". -/// Only ASCII letters and digits are considered for boundaries. -pub fn decamelize(text: String) -> Result(String, SlugifyError) { - text - |> string.to_graphemes - |> decamelize_graphemes([]) - |> string.join("") - |> Ok -} - -fn decamelize_graphemes( - graphemes: List(String), - acc: List(String), -) -> List(String) { - case graphemes { - [a, b, ..rest] -> { - case is_lower_or_digit(a) && is_upper(b) { - True -> decamelize_graphemes([b, ..rest], [" ", a, ..acc]) - False -> - case rest { - [c, ..] -> - case is_upper(a) && is_upper(b) && is_lower_or_digit(c) { - True -> decamelize_graphemes([b, ..rest], [" ", a, ..acc]) - False -> decamelize_graphemes([b, ..rest], [a, ..acc]) - } - [] -> decamelize_graphemes([b, ..rest], [a, ..acc]) - } - } - } - [a] -> list.reverse([a, ..acc]) - [] -> list.reverse(acc) - } -} - -fn is_upper(char: String) -> Bool { - case string.to_utf_codepoints(char) { - [codepoint] -> { - let code = string.utf_codepoint_to_int(codepoint) - code >= 65 && code <= 90 - } - _ -> False - } -} - -fn is_lower_or_digit(char: String) -> Bool { - case string.to_utf_codepoints(char) { - [codepoint] -> { - let code = string.utf_codepoint_to_int(codepoint) - { code >= 97 && code <= 122 } || { code >= 48 && code <= 57 } - } - _ -> False - } -} - -pub fn apply_custom_replacements( - text: String, - replacements: List(#(String, String)), -) -> Result(String, SlugifyError) { - apply_replacements_helper(text, replacements) |> Ok -} - -fn apply_replacements_helper( - text: String, - replacements: List(#(String, String)), -) -> String { - case replacements { - [] -> text - [#(find, replace), ..rest] -> { - let updated = string.replace(text, find, replace) - apply_replacements_helper(updated, rest) - } - } -} - -pub fn filter_stop_words( - text: String, - stop_words: List(String), - separator: String, -) -> Result(String, SlugifyError) { - // Splitting on "" would split into graphemes and delete every letter - // that happens to match a stop word, so an empty separator is a no-op. - case list.is_empty(stop_words) || separator == "" { - True -> Ok(text) - False -> { - let lowered_stop_words = list.map(stop_words, string.lowercase) - let words = string.split(text, separator) - let filtered_words = - filter_stop_words_helper(words, lowered_stop_words, []) - string.join(filtered_words, separator) |> Ok - } - } -} - -fn filter_stop_words_helper( - words: List(String), - lowered_stop_words: List(String), - acc: List(String), -) -> List(String) { - case words { - [] -> list.reverse(acc) - [word, ..rest] -> { - case list.contains(lowered_stop_words, string.lowercase(word)) { - True -> filter_stop_words_helper(rest, lowered_stop_words, acc) - False -> - filter_stop_words_helper(rest, lowered_stop_words, [word, ..acc]) - } - } - } -} diff --git a/src/glugify/internal/validators.gleam b/src/glugify/internal/validators.gleam deleted file mode 100644 index ac839ed..0000000 --- a/src/glugify/internal/validators.gleam +++ /dev/null @@ -1,9 +0,0 @@ -import gleam/string -import glugify/errors.{type SlugifyError} - -pub fn validate_input(text: String) -> Result(String, SlugifyError) { - case string.trim(text) { - "" -> Error(errors.EmptyInput) - trimmed -> Ok(trimmed) - } -} diff --git a/src/glugify/locale.gleam b/src/glugify/locale.gleam index 3789d22..fbd4a40 100644 --- a/src/glugify/locale.gleam +++ b/src/glugify/locale.gleam @@ -1,24 +1,3 @@ -/// Locales for language-specific transliteration rules. -/// -/// The default transliteration maps each character to its closest single -/// ASCII letter (`ä` -> `a`). Some languages have their own romanization -/// conventions, which a locale enables: -/// -/// - `German`: `ä` -> `ae`, `ö` -> `oe`, `ü` -> `ue` -/// - `Danish` / `Norwegian`: `æ` -> `ae`, `ø` -> `oe`, `å` -> `aa` -/// - `Swedish` and `Turkish`: same as the default mapping; provided so -/// the intent can be stated explicitly in code -/// -/// ## Examples -/// -/// ```gleam -/// import glugify/config -/// import glugify/locale -/// -/// config.default() -/// |> config.with_locale(locale.German) -/// // "Über München" -> "ueber-muenchen" -/// ``` pub type Locale { Default German diff --git a/src/glugify/slugger.gleam b/src/glugify/slugger.gleam index 064b5e0..1b617a2 100644 --- a/src/glugify/slugger.gleam +++ b/src/glugify/slugger.gleam @@ -1,24 +1,3 @@ -//// A stateful slugger that guarantees unique slugs across a batch of -//// inputs — the building block for tables of contents, static site -//// generators and CMS imports, where two posts titled "Hello" must not -//// share a URL. -//// -//// The state is an immutable value threaded through each call, so it -//// works naturally in folds and works identically on the Erlang and -//// JavaScript targets: -//// -//// ```gleam -//// import glugify/slugger -//// -//// let s = slugger.new() -//// let #(s, a) = slugger.slug(s, "Hello World") -//// let #(s, b) = slugger.slug(s, "Hello World") -//// let #(_, c) = slugger.slug(s, "Hello World") -//// // a -> "hello-world" -//// // b -> "hello-world-1" -//// // c -> "hello-world-2" -//// ``` - import gleam/dict.{type Dict} import gleam/int import gleam/result @@ -27,52 +6,18 @@ import glugify import glugify/config.{type Config} import glugify/errors.{type SlugifyError} -/// Tracks which slugs have been handed out. Create one with `new` and -/// thread it through `slug` or `slug_with` calls. pub opaque type Slugger { Slugger(taken: Set(String), counts: Dict(String, Int)) } -/// Creates a fresh slugger with no slugs taken. pub fn new() -> Slugger { Slugger(taken: set.new(), counts: dict.new()) } -/// Slugifies `text` with the default configuration and makes the result -/// unique against everything this slugger has produced before, by -/// appending `-1`, `-2`, ... to duplicates. -/// -/// Unlike github-slugger, a suffixed slug never collides with a slug -/// produced from genuinely suffixed input: `"foo"`, `"foo"`, `"foo-1"` -/// yields `"foo"`, `"foo-1"`, `"foo-1-1"`. -/// -/// Empty results (e.g. from symbol-only input) bypass uniqueness and are -/// returned as-is. -/// -/// ## Examples -/// -/// ```gleam -/// let #(slugger, first) = slug(new(), "My Post") -/// let #(_, second) = slug(slugger, "My Post") -/// // first -> "my-post" -/// // second -> "my-post-1" -/// ``` pub fn slug(slugger: Slugger, text: String) -> #(Slugger, String) { - let base = glugify.slugify(text) - unique(slugger, base) + unique(slugger, glugify.slugify(text)) } -/// Like `slug`, but slugifies with a custom configuration. -/// -/// ## Examples -/// -/// ```gleam -/// import glugify/config -/// -/// let cfg = config.default() |> config.with_separator("_") -/// slug_with(new(), "My Post", cfg) -/// // -> Ok(#(slugger, "my_post")) -/// ``` pub fn slug_with( slugger: Slugger, text: String, @@ -83,22 +28,20 @@ pub fn slug_with( } fn unique(slugger: Slugger, base: String) -> #(Slugger, String) { - case base { - "" -> #(slugger, "") - _ -> - case set.contains(slugger.taken, base) { - False -> #( - Slugger(..slugger, taken: set.insert(slugger.taken, base)), - base, - ) - True -> { - let next = case dict.get(slugger.counts, base) { - Ok(count) -> count - Error(Nil) -> 1 - } - find_free(slugger, base, next) - } - } + case base, set.contains(slugger.taken, base) { + "", _ -> #(slugger, "") + _, False -> #( + Slugger(..slugger, taken: set.insert(slugger.taken, base)), + base, + ) + _, True -> find_free(slugger, base, next_suffix(slugger, base)) + } +} + +fn next_suffix(slugger: Slugger, base: String) -> Int { + case dict.get(slugger.counts, base) { + Ok(count) -> count + Error(Nil) -> 1 } } diff --git a/src/glugify/transformations.gleam b/src/glugify/transformations.gleam index 294463a..d4d71ed 100644 --- a/src/glugify/transformations.gleam +++ b/src/glugify/transformations.gleam @@ -3,37 +3,9 @@ import gleam/result import glugify/errors.{type SlugifyError} import glugify/unicode -/// A transformation function that processes text and may fail with an error. -/// -/// Transformations can be composed together to create complex processing pipelines -/// while preserving type safety and error handling. -/// -/// ## Examples -/// -/// ```gleam -/// let my_transform: Transformation = fn(text) { -/// text |> string.uppercase |> Ok -/// } -/// ``` pub type Transformation = fn(String) -> Result(String, SlugifyError) -/// Composes multiple transformations into a single transformation. -/// -/// Transformations are applied in order from left to right. -/// If any transformation fails, the entire composition fails. -/// -/// ## Examples -/// -/// ```gleam -/// let pipeline = compose([ -/// normalize_unicode(), -/// transliterate_to_ascii() -/// ]) -/// -/// pipeline("café") -/// // -> Ok("cafe") -/// ``` pub fn compose(transformations: List(Transformation)) -> Transformation { fn(input: String) -> Result(String, SlugifyError) { list.fold(transformations, Ok(input), fn(acc, transform) { @@ -42,34 +14,10 @@ pub fn compose(transformations: List(Transformation)) -> Transformation { } } -/// Creates a transformation that normalizes Unicode text. -/// -/// Currently this is a no-op transformation that passes text through unchanged. -/// In the future, this could implement Unicode normalization forms (NFC, NFD, etc.). -/// -/// ## Examples -/// -/// ```gleam -/// let normalizer = normalize_unicode() -/// normalizer("text") -/// // -> Ok("text") -/// ``` pub fn normalize_unicode() -> Transformation { fn(text: String) -> Result(String, SlugifyError) { Ok(text) } } -/// Creates a transformation that transliterates Unicode to ASCII. -/// -/// This transformation converts accented characters and symbols to their -/// ASCII equivalents. Characters that cannot be transliterated will cause an error. -/// -/// ## Examples -/// -/// ```gleam -/// let transliterator = transliterate_to_ascii() -/// transliterator("café") -/// // -> Ok("cafe") -/// ``` pub fn transliterate_to_ascii() -> Transformation { unicode.transliterate_text } diff --git a/src/glugify/unicode.gleam b/src/glugify/unicode.gleam index 1620151..c2c0ece 100644 --- a/src/glugify/unicode.gleam +++ b/src/glugify/unicode.gleam @@ -1,3 +1,4 @@ +import gleam/dict.{type Dict} import gleam/list import gleam/result import gleam/string @@ -5,98 +6,27 @@ import glugify/errors.{type SlugifyError} import glugify/internal/char_maps import glugify/locale.{type Locale} -/// Transliterates Unicode text to ASCII equivalents. -/// -/// This function converts accented characters, Cyrillic, Greek, typographic -/// punctuation and common symbols to their ASCII equivalents. Decomposed -/// (NFD) characters are handled by stripping combining marks and mapping the -/// base character. Characters with no known mapping (such as emoji) are -/// stripped from the output. -/// -/// ## Examples -/// -/// ```gleam -/// transliterate_text("café") -/// // -> Ok("cafe") -/// -/// transliterate_text("naïve") -/// // -> Ok("naive") -/// -/// transliterate_text("Résumé 🚀") -/// // -> Ok("Resume ") -/// ``` pub fn transliterate_text(text: String) -> Result(String, SlugifyError) { transliterate_text_with(text, locale.Default, []) } -/// Transliterates Unicode text to ASCII using locale-specific rules, -/// keeping the graphemes in `ignore` verbatim. -/// -/// Locale rules take precedence over the general tables, so with -/// `locale.German` the text "Über" becomes "Ueber" rather than "Uber". -/// -/// ## Examples -/// -/// ```gleam -/// import glugify/locale -/// -/// transliterate_text_with("Über München", locale.German, []) -/// // -> Ok("Ueber Muenchen") -/// -/// transliterate_text_with("嗨 hello", locale.Default, ["嗨"]) -/// // -> Ok("嗨 hello") -/// ``` pub fn transliterate_text_with( text: String, locale: Locale, ignore: List(String), ) -> Result(String, SlugifyError) { + let table = char_maps.table(locale) text |> string.to_graphemes - |> transliterate_graphemes(locale, ignore, []) - |> result.map(string.join(_, "")) + |> list.map(fn(grapheme) { transliterate_grapheme(grapheme, table, ignore) }) + |> string.join("") + |> Ok } -/// Validates that text contains only ASCII characters. -/// -/// This function checks that all characters in the input are within -/// the printable ASCII range (32-126). Non-ASCII characters will cause -/// an error to be returned. -/// -/// ## Examples -/// -/// ```gleam -/// validate_ascii_only("hello world") -/// // -> Ok("hello world") -/// -/// validate_ascii_only("café") -/// // -> Error(TransliterationFailed("é")) -/// ``` pub fn validate_ascii_only(text: String) -> Result(String, SlugifyError) { - text - |> string.to_graphemes - |> validate_ascii_graphemes([]) - |> result.map(fn(_) { text }) + validate_ascii_or_unicode(text, False, []) } -/// Validates text based on Unicode allowance settings. -/// -/// If `allow_unicode` is `True`, all text is accepted. -/// If `allow_unicode` is `False`, only ASCII text and graphemes listed -/// in `ignore` are accepted. -/// -/// ## Examples -/// -/// ```gleam -/// validate_ascii_or_unicode("café", True, []) -/// // -> Ok("café") -/// -/// validate_ascii_or_unicode("café", False, []) -/// // -> Error(TransliterationFailed("é")) -/// -/// validate_ascii_or_unicode("café", False, ["é"]) -/// // -> Ok("café") -/// ``` pub fn validate_ascii_or_unicode( text: String, allow_unicode: Bool, @@ -116,114 +46,98 @@ fn validate_ascii_graphemes( graphemes: List(String), ignore: List(String), ) -> Result(Nil, SlugifyError) { - case graphemes { - [] -> Ok(Nil) - [grapheme, ..rest] -> { - case is_ascii_safe(grapheme) || list.contains(ignore, grapheme) { - True -> validate_ascii_graphemes(rest, ignore) - False -> Error(errors.TransliterationFailed(grapheme)) - } - } + case list.find(graphemes, fn(g) { !is_permitted(g, ignore) }) { + Ok(grapheme) -> Error(errors.TransliterationFailed(grapheme)) + Error(Nil) -> Ok(Nil) } } -fn transliterate_graphemes( - graphemes: List(String), - locale: Locale, +fn is_permitted(grapheme: String, ignore: List(String)) -> Bool { + is_ascii_safe(grapheme) || list.contains(ignore, grapheme) +} + +fn transliterate_grapheme( + grapheme: String, + table: Dict(String, String), ignore: List(String), - acc: List(String), -) -> Result(List(String), SlugifyError) { - case graphemes { - [] -> Ok(list.reverse(acc)) - [grapheme, ..rest] -> { - case list.contains(ignore, grapheme) { - True -> transliterate_graphemes(rest, locale, ignore, [grapheme, ..acc]) - False -> - case lookup_with_locale(grapheme, locale) { - Ok(replacement) -> - transliterate_graphemes(rest, locale, ignore, [replacement, ..acc]) - Error(_) -> { - case is_ascii_safe(grapheme) { - True -> - transliterate_graphemes(rest, locale, ignore, [ - grapheme, - ..acc - ]) - False -> - transliterate_graphemes(rest, locale, ignore, [ - transliterate_codepoints(grapheme, locale), - ..acc - ]) - } - } - } - } - } +) -> String { + case list.contains(ignore, grapheme), dict.get(table, grapheme) { + True, _ -> grapheme + False, Ok(replacement) -> replacement + False, Error(Nil) -> transliterate_unmapped(grapheme, table) } } -fn lookup_with_locale(grapheme: String, locale: Locale) -> Result(String, Nil) { - case char_maps.lookup_locale(grapheme, locale) { - Ok(replacement) -> Ok(replacement) - Error(Nil) -> char_maps.lookup(grapheme) +fn transliterate_unmapped( + grapheme: String, + table: Dict(String, String), +) -> String { + case is_ascii_safe(grapheme) { + True -> grapheme + False -> transliterate_codepoints(grapheme, table) } } -/// Fallback for graphemes that have no direct mapping: decompose into -/// codepoints, drop combining marks and invisible joiners, and map each -/// remaining codepoint individually. Handles NFD (decomposed) input such -/// as "e" followed by U+0301. Codepoints with no mapping are stripped. -fn transliterate_codepoints(grapheme: String, locale: Locale) -> String { +fn transliterate_codepoints( + grapheme: String, + table: Dict(String, String), +) -> String { grapheme |> string.to_utf_codepoints - |> list.filter_map(fn(codepoint) { - let code = string.utf_codepoint_to_int(codepoint) - case is_ignorable_codepoint(code) { - True -> Error(Nil) - False -> { - let char = string.from_utf_codepoints([codepoint]) - case lookup_with_locale(char, locale) { - Ok(replacement) -> Ok(replacement) - Error(_) -> - case code >= 32 && code <= 126 { - True -> Ok(char) - False -> Error(Nil) - } - } - } - } - }) + |> list.filter_map(fn(codepoint) { map_codepoint(codepoint, table) }) |> string.join("") } -/// Combining marks (U+0300-U+036F), Hebrew cantillation and niqqud -/// (U+0591-U+05C7), Arabic tashkeel (U+064B-U+0652), zero-width -/// joiners/non-joiners and bidirectional marks (U+200C-U+200F), and -/// variation selectors (U+FE00-U+FE0F) carry no slug content of their own. +fn map_codepoint( + codepoint: UtfCodepoint, + table: Dict(String, String), +) -> Result(String, Nil) { + let code = string.utf_codepoint_to_int(codepoint) + let char = string.from_utf_codepoints([codepoint]) + case is_ignorable_codepoint(code), dict.get(table, char) { + True, _ -> Error(Nil) + False, Ok(replacement) -> Ok(replacement) + False, Error(Nil) -> keep_when_printable_ascii(char, code) + } +} + +fn keep_when_printable_ascii(char: String, code: Int) -> Result(String, Nil) { + case is_printable_ascii(code) { + True -> Ok(char) + False -> Error(Nil) + } +} + +const combining_marks = #(0x0300, 0x036F) + +const hebrew_cantillation_and_niqqud = #(0x0591, 0x05C7) + +const arabic_tashkeel = #(0x064B, 0x0652) + +const joiners_and_bidi_marks = #(0x200C, 0x200F) + +const variation_selectors = #(0xFE00, 0xFE0F) + +const content_free_ranges: List(#(Int, Int)) = [ + combining_marks, hebrew_cantillation_and_niqqud, arabic_tashkeel, + joiners_and_bidi_marks, variation_selectors, +] + fn is_ignorable_codepoint(code: Int) -> Bool { - { code >= 0x0300 && code <= 0x036F } - || { code >= 0x0591 && code <= 0x05C7 } - || { code >= 0x064B && code <= 0x0652 } - || { code >= 0x200C && code <= 0x200F } - || { code >= 0xFE00 && code <= 0xFE0F } + list.any(content_free_ranges, fn(range) { within(code, range) }) +} + +fn within(code: Int, range: #(Int, Int)) -> Bool { + code >= range.0 && code <= range.1 +} + +fn is_printable_ascii(code: Int) -> Bool { + code >= 32 && code <= 126 } fn is_ascii_safe(char: String) -> Bool { - case string.length(char) { - 1 -> { - case string.pop_grapheme(char) { - Ok(#(grapheme, "")) -> { - case string.to_utf_codepoints(grapheme) { - [codepoint] -> { - let code = string.utf_codepoint_to_int(codepoint) - code >= 32 && code <= 126 - } - _ -> False - } - } - _ -> False - } - } + case string.to_utf_codepoints(char) { + [codepoint] -> is_printable_ascii(string.utf_codepoint_to_int(codepoint)) _ -> False } } diff --git a/src/glugify_ffi.erl b/src/glugify_ffi.erl index 4a21a8a..315678e 100644 --- a/src/glugify_ffi.erl +++ b/src/glugify_ffi.erl @@ -1,17 +1,6 @@ -module(glugify_ffi). -export([github_clean/1]). -%% Removes every character GitHub strips when turning a heading into an -%% anchor, then turns each remaining space into a hyphen. The kept set -%% mirrors github-slugger's generated regex: Unicode letters (L), marks -%% (M), decimal and letter numbers (Nd, Nl), connector punctuation (Pc), -%% the circled and squared Latin letters that are Alphabetic without -%% being letters (U+24B6-24E9, U+1F130-1F189), plus space and hyphen. -%% -%% The space replacement happens here, at the codepoint level, rather -%% than in Gleam: string:replace/3 is grapheme-aware, so a space carrying -%% a combining mark would not match a lone " " pattern, and the -%% JavaScript target would disagree. github_clean(Text) -> Stripped = re:replace( Text, diff --git a/tasks/000-green-the-repo.md b/tasks/000-green-the-repo.md new file mode 100644 index 0000000..e070068 --- /dev/null +++ b/tasks/000-green-the-repo.md @@ -0,0 +1,19 @@ +# Bring the Gleam package through the gate + +Nothing about the library's behaviour may change. Every public API, return value, and error contract stays exactly as it is; the Gherkin scenarios must pin them down before any code moves. + +Make `marestail gate --tier full` pass for the Gleam side: + +- `gleam test` green for everything under `test/` (gleeunit), with 100% line coverage measured via Erlang `cover` on the compiled BEAMs (mapped back to `.gleam` through the compiler's `-file` attributes). +- Every function at CRAP 4 or below (glance complexity joined to that cover data). +- No comments (`//` or `///`) anywhere under `src/`. Where a comment carries knowledge, turn it into a name, a small function, or a test. +- `gleam format --check` and `gleam build --warnings-as-errors` clean. +- Dependency contracts kept: no import cycles among `src/` modules. Same-package use of `glugify/internal/*` is allowed (Gleam already blocks other packages from importing them). +- No unused private functions under `src/`. +- No pass-through functions that only forward their own arguments. +- Sonar: quality gate green, zero open issues, zero hotspots to review, zero duplication. +- Mutation is skipped for Gleam (muex mutates Elixir/Erlang AST, not Gleam); the hardener still judges the diff and the feature coverage. + +Out of scope: `docs/`, `dev/`, Hex publishing, and dependency upgrades beyond what the tools need. + +Note on Gleam internals: modules under `src/glugify/internal/` are package-private by language rule; other packages cannot import them. Same-package imports of `glugify/internal/*` are allowed and expected. diff --git a/tasks/README.md b/tasks/README.md new file mode 100644 index 0000000..af57e2f --- /dev/null +++ b/tasks/README.md @@ -0,0 +1,13 @@ +# Writing tasks + +One task is one pipeline run: specifier, critic, coder, cleaner, architect, hardener, QA. Write it so that run ends with something a user could do that they could not do before. + +- **Slice vertically.** Name the user and the outcome: "a donor gives 50 zł by card through the new widget". Reach every layer that outcome needs and no wider. "Build the amounts component" is a layer, not a task. +- **Keep it thin.** One payment method, one variant, one route. The next slice adds the next one. If the task needs a list of sub-features, it is several tasks. +- **Say what must not change.** Existing routes, contracts, and behaviour the slice touches. The specifier pins these in scenarios. +- **Give the values.** Amounts, currencies, error messages, locales. Concrete values become concrete scenarios. +- **Put the rules in the gate, not the task.** Coverage, complexity, comments, and Sonar are already enforced. Mention a rule only when this task needs a different one. +- **Refactors are the exception.** A task that changes no behaviour says so in the first line and freezes everything; the critic will not demand a user-visible outcome. +- **Order slices in a file** when several belong together, and run them one after another; the architect reshapes modules between them. + +Name files `NNN-short-name.md`. The specifier writes `features/short-name.feature` and `qa/short-name.md` to match. diff --git a/test/char_table_test.gleam b/test/char_table_test.gleam new file mode 100644 index 0000000..9135c7f --- /dev/null +++ b/test/char_table_test.gleam @@ -0,0 +1,584 @@ +import gleam/list +import gleeunit/should +import glugify/locale +import glugify/unicode + +fn check_rows(rows: List(#(String, String)), locale: locale.Locale) -> Nil { + list.each(rows, fn(row) { + unicode.transliterate_text_with(row.0, locale, []) + |> should.equal(Ok(row.1)) + }) +} + +pub fn transliteration_table_row_count_test() { + let counts = + list.map( + [ + latin_rows, cyrillic_rows, greek_rows, arabic_rows, hebrew_rows, + punctuation_rows, symbol_rows, german_rows, danish_rows, + ], + list.length, + ) + list.fold(counts, 0, fn(a, b) { a + b }) + |> should.equal(497) +} + +pub fn transliteration_table_keeps_latin_mappings_test() { + check_rows(latin_rows, locale.Default) +} + +pub fn transliteration_table_keeps_cyrillic_mappings_test() { + check_rows(cyrillic_rows, locale.Default) +} + +pub fn transliteration_table_keeps_greek_mappings_test() { + check_rows(greek_rows, locale.Default) +} + +pub fn transliteration_table_keeps_arabic_mappings_test() { + check_rows(arabic_rows, locale.Default) +} + +pub fn transliteration_table_keeps_hebrew_mappings_test() { + check_rows(hebrew_rows, locale.Default) +} + +pub fn transliteration_table_keeps_punctuation_mappings_test() { + check_rows(punctuation_rows, locale.Default) +} + +pub fn transliteration_table_keeps_symbol_mappings_test() { + check_rows(symbol_rows, locale.Default) +} + +pub fn transliteration_table_keeps_german_mappings_test() { + check_rows(german_rows, locale.German) +} + +pub fn transliteration_table_keeps_danish_mappings_test() { + check_rows(danish_rows, locale.Danish) +} + +const latin_rows: List(#(String, String)) = [ + #("à", "a"), + #("á", "a"), + #("â", "a"), + #("ã", "a"), + #("ä", "a"), + #("å", "a"), + #("ā", "a"), + #("ă", "a"), + #("ą", "a"), + #("À", "A"), + #("Á", "A"), + #("Â", "A"), + #("Ã", "A"), + #("Ä", "A"), + #("Å", "A"), + #("Ā", "A"), + #("Ă", "A"), + #("Ą", "A"), + #("è", "e"), + #("é", "e"), + #("ê", "e"), + #("ë", "e"), + #("ē", "e"), + #("ĕ", "e"), + #("ė", "e"), + #("ę", "e"), + #("ě", "e"), + #("È", "E"), + #("É", "E"), + #("Ê", "E"), + #("Ë", "E"), + #("Ē", "E"), + #("Ĕ", "E"), + #("Ė", "E"), + #("Ę", "E"), + #("Ě", "E"), + #("ì", "i"), + #("í", "i"), + #("î", "i"), + #("ï", "i"), + #("ī", "i"), + #("ĭ", "i"), + #("į", "i"), + #("ı", "i"), + #("Ì", "I"), + #("Í", "I"), + #("Î", "I"), + #("Ï", "I"), + #("Ī", "I"), + #("Ĭ", "I"), + #("Į", "I"), + #("İ", "I"), + #("ò", "o"), + #("ó", "o"), + #("ô", "o"), + #("õ", "o"), + #("ö", "o"), + #("ø", "o"), + #("ō", "o"), + #("ŏ", "o"), + #("ő", "o"), + #("Ò", "O"), + #("Ó", "O"), + #("Ô", "O"), + #("Õ", "O"), + #("Ö", "O"), + #("Ø", "O"), + #("Ō", "O"), + #("Ŏ", "O"), + #("Ő", "O"), + #("ù", "u"), + #("ú", "u"), + #("û", "u"), + #("ü", "u"), + #("ū", "u"), + #("ŭ", "u"), + #("ů", "u"), + #("ű", "u"), + #("ų", "u"), + #("Ù", "U"), + #("Ú", "U"), + #("Û", "U"), + #("Ü", "U"), + #("Ū", "U"), + #("Ŭ", "U"), + #("Ů", "U"), + #("Ű", "U"), + #("Ų", "U"), + #("ý", "y"), + #("ÿ", "y"), + #("ŷ", "y"), + #("Ý", "Y"), + #("Ÿ", "Y"), + #("Ŷ", "Y"), + #("ç", "c"), + #("ć", "c"), + #("ĉ", "c"), + #("ċ", "c"), + #("č", "c"), + #("Ç", "C"), + #("Ć", "C"), + #("Ĉ", "C"), + #("Ċ", "C"), + #("Č", "C"), + #("ñ", "n"), + #("ń", "n"), + #("ņ", "n"), + #("ň", "n"), + #("Ñ", "N"), + #("Ń", "N"), + #("Ņ", "N"), + #("Ň", "N"), + #("ß", "ss"), + #("æ", "ae"), + #("Æ", "AE"), + #("œ", "oe"), + #("Œ", "OE"), + #("ð", "d"), + #("Ð", "D"), + #("þ", "th"), + #("Þ", "Th"), + #("đ", "d"), + #("ď", "d"), + #("Đ", "D"), + #("Ď", "D"), + #("ġ", "g"), + #("ğ", "g"), + #("ĝ", "g"), + #("ģ", "g"), + #("Ġ", "G"), + #("Ğ", "G"), + #("Ĝ", "G"), + #("Ģ", "G"), + #("ĥ", "h"), + #("ħ", "h"), + #("Ĥ", "H"), + #("Ħ", "H"), + #("ĵ", "j"), + #("Ĵ", "J"), + #("ķ", "k"), + #("Ķ", "K"), + #("ĺ", "l"), + #("ļ", "l"), + #("ľ", "l"), + #("ŀ", "l"), + #("ł", "l"), + #("Ĺ", "L"), + #("Ļ", "L"), + #("Ľ", "L"), + #("Ŀ", "L"), + #("Ł", "L"), + #("ŕ", "r"), + #("ŗ", "r"), + #("ř", "r"), + #("Ŕ", "R"), + #("Ŗ", "R"), + #("Ř", "R"), + #("ś", "s"), + #("ŝ", "s"), + #("ş", "s"), + #("š", "s"), + #("ș", "s"), + #("Ś", "S"), + #("Ŝ", "S"), + #("Ş", "S"), + #("Š", "S"), + #("Ș", "S"), + #("ţ", "t"), + #("ť", "t"), + #("ŧ", "t"), + #("ț", "t"), + #("Ţ", "T"), + #("Ť", "T"), + #("Ŧ", "T"), + #("Ț", "T"), + #("ŵ", "w"), + #("Ŵ", "W"), + #("ź", "z"), + #("ż", "z"), + #("ž", "z"), + #("Ź", "Z"), + #("Ż", "Z"), + #("Ž", "Z"), +] + +const cyrillic_rows: List(#(String, String)) = [ + #("а", "a"), + #("б", "b"), + #("в", "v"), + #("г", "g"), + #("д", "d"), + #("е", "e"), + #("ё", "yo"), + #("ж", "zh"), + #("з", "z"), + #("и", "i"), + #("й", "y"), + #("к", "k"), + #("л", "l"), + #("м", "m"), + #("н", "n"), + #("о", "o"), + #("п", "p"), + #("р", "r"), + #("с", "s"), + #("т", "t"), + #("у", "u"), + #("ф", "f"), + #("х", "h"), + #("ц", "ts"), + #("ч", "ch"), + #("ш", "sh"), + #("щ", "shch"), + #("ъ", ""), + #("ы", "y"), + #("ь", ""), + #("э", "e"), + #("ю", "yu"), + #("я", "ya"), + #("є", "ye"), + #("і", "i"), + #("ї", "yi"), + #("ґ", "g"), + #("ў", "u"), + #("А", "A"), + #("Б", "B"), + #("В", "V"), + #("Г", "G"), + #("Д", "D"), + #("Е", "E"), + #("Ё", "Yo"), + #("Ж", "Zh"), + #("З", "Z"), + #("И", "I"), + #("Й", "Y"), + #("К", "K"), + #("Л", "L"), + #("М", "M"), + #("Н", "N"), + #("О", "O"), + #("П", "P"), + #("Р", "R"), + #("С", "S"), + #("Т", "T"), + #("У", "U"), + #("Ф", "F"), + #("Х", "H"), + #("Ц", "Ts"), + #("Ч", "Ch"), + #("Ш", "Sh"), + #("Щ", "Shch"), + #("Ъ", ""), + #("Ы", "Y"), + #("Ь", ""), + #("Э", "E"), + #("Ю", "Yu"), + #("Я", "Ya"), + #("Є", "Ye"), + #("І", "I"), + #("Ї", "Yi"), + #("Ґ", "G"), + #("Ў", "U"), +] + +const greek_rows: List(#(String, String)) = [ + #("α", "a"), + #("ά", "a"), + #("β", "v"), + #("γ", "g"), + #("δ", "d"), + #("ε", "e"), + #("έ", "e"), + #("ζ", "z"), + #("η", "i"), + #("ή", "i"), + #("θ", "th"), + #("ι", "i"), + #("ί", "i"), + #("ϊ", "i"), + #("ΐ", "i"), + #("κ", "k"), + #("λ", "l"), + #("μ", "m"), + #("ν", "n"), + #("ξ", "ks"), + #("ο", "o"), + #("ό", "o"), + #("π", "p"), + #("ρ", "r"), + #("σ", "s"), + #("ς", "s"), + #("τ", "t"), + #("υ", "y"), + #("ύ", "y"), + #("ϋ", "y"), + #("ΰ", "y"), + #("φ", "f"), + #("χ", "x"), + #("ψ", "ps"), + #("ω", "o"), + #("ώ", "o"), + #("Α", "A"), + #("Ά", "A"), + #("Β", "V"), + #("Γ", "G"), + #("Δ", "D"), + #("Ε", "E"), + #("Έ", "E"), + #("Ζ", "Z"), + #("Η", "I"), + #("Ή", "I"), + #("Θ", "Th"), + #("Ι", "I"), + #("Ί", "I"), + #("Κ", "K"), + #("Λ", "L"), + #("Μ", "M"), + #("Ν", "N"), + #("Ξ", "Ks"), + #("Ο", "O"), + #("Ό", "O"), + #("Π", "P"), + #("Ρ", "R"), + #("Σ", "S"), + #("Τ", "T"), + #("Υ", "Y"), + #("Ύ", "Y"), + #("Φ", "F"), + #("Χ", "X"), + #("Ψ", "Ps"), + #("Ω", "O"), + #("Ώ", "O"), +] + +const arabic_rows: List(#(String, String)) = [ + #("ا", "a"), + #("أ", "a"), + #("آ", "a"), + #("ى", "a"), + #("إ", "i"), + #("ء", "e"), + #("ئ", "e"), + #("ؤ", "w"), + #("ب", "b"), + #("ت", "t"), + #("ث", "th"), + #("ج", "j"), + #("ح", "h"), + #("خ", "kh"), + #("د", "d"), + #("ذ", "dh"), + #("ر", "r"), + #("ز", "z"), + #("س", "s"), + #("ش", "sh"), + #("ص", "s"), + #("ض", "d"), + #("ط", "t"), + #("ظ", "z"), + #("ع", "a"), + #("غ", "gh"), + #("ف", "f"), + #("ق", "q"), + #("ك", "k"), + #("ک", "k"), + #("ل", "l"), + #("م", "m"), + #("ن", "n"), + #("ه", "h"), + #("ة", "h"), + #("و", "w"), + #("ي", "y"), + #("ی", "y"), + #("پ", "p"), + #("چ", "ch"), + #("ژ", "zh"), + #("گ", "g"), + #("٠", "0"), + #("۰", "0"), + #("١", "1"), + #("۱", "1"), + #("٢", "2"), + #("۲", "2"), + #("٣", "3"), + #("۳", "3"), + #("٤", "4"), + #("۴", "4"), + #("٥", "5"), + #("۵", "5"), + #("٦", "6"), + #("۶", "6"), + #("٧", "7"), + #("۷", "7"), + #("٨", "8"), + #("۸", "8"), + #("٩", "9"), + #("۹", "9"), +] + +const hebrew_rows: List(#(String, String)) = [ + #("א", "a"), + #("ב", "b"), + #("ג", "g"), + #("ד", "d"), + #("ה", "h"), + #("ו", "v"), + #("ז", "z"), + #("ח", "h"), + #("ט", "t"), + #("י", "y"), + #("כ", "k"), + #("ך", "k"), + #("ל", "l"), + #("מ", "m"), + #("ם", "m"), + #("נ", "n"), + #("ן", "n"), + #("ס", "s"), + #("ע", "a"), + #("פ", "p"), + #("ף", "p"), + #("צ", "ts"), + #("ץ", "ts"), + #("ק", "k"), + #("ר", "r"), + #("ש", "sh"), + #("ת", "t"), +] + +const punctuation_rows: List(#(String, String)) = [ + #("'", ""), + #("’", ""), + #("‘", ""), + #("‚", ""), + #("ʼ", ""), + #("ʻ", ""), + #("“", " "), + #("”", " "), + #("„", " "), + #("‟", " "), + #("«", " "), + #("»", " "), + #("‹", " "), + #("›", " "), + #("′", " "), + #("″", " "), + #("–", " "), + #("—", " "), + #("―", " "), + #("‒", " "), + #("‐", " "), + #("‑", " "), + #("…", " "), + #("•", " "), + #("·", " "), + #("∙", " "), + #("¡", " "), + #("¿", " "), + #("§", " "), + #("¶", " "), + #("†", " "), + #("‡", " "), + #("\u{00A0}", " "), + #("\u{2000}", " "), + #("\u{2001}", " "), + #("\u{2002}", " "), + #("\u{2003}", " "), + #("\u{2004}", " "), + #("\u{2005}", " "), + #("\u{2006}", " "), + #("\u{2007}", " "), + #("\u{2008}", " "), + #("\u{2009}", " "), + #("\u{200A}", " "), + #("\u{200B}", " "), + #("\u{202F}", " "), + #("\u{3000}", " "), +] + +const symbol_rows: List(#(String, String)) = [ + #("&", " and "), + #("@", " at "), + #("%", " percent "), + #("$", " dollar "), + #("€", " euro "), + #("£", " pound "), + #("¥", " yen "), + #("¢", " cent "), + #("₹", " rupee "), + #("₩", " won "), + #("₽", " ruble "), + #("₺", " lira "), + #("₫", " dong "), + #("฿", " baht "), + #("©", " c "), + #("®", " r "), + #("™", " tm "), + #("°", " degrees "), + #("№", " no "), + #("×", " x "), + #("÷", " "), + #("±", " plus "), + #("µ", "u"), +] + +const german_rows: List(#(String, String)) = [ + #("ä", "ae"), + #("ö", "oe"), + #("ü", "ue"), + #("Ä", "Ae"), + #("Ö", "Oe"), + #("Ü", "Ue"), +] + +const danish_rows: List(#(String, String)) = [ + #("æ", "ae"), + #("ø", "oe"), + #("å", "aa"), + #("Æ", "Ae"), + #("Ø", "Oe"), + #("Å", "Aa"), +] diff --git a/test/green_the_repo_test.gleam b/test/green_the_repo_test.gleam new file mode 100644 index 0000000..941cf39 --- /dev/null +++ b/test/green_the_repo_test.gleam @@ -0,0 +1,591 @@ +import gleam/option +import gleeunit/should +import glugify +import glugify/anchor +import glugify/config +import glugify/errors +import glugify/locale +import glugify/slugger +import glugify/transformations +import glugify/unicode + +pub fn slugify_returns_a_plain_string_test() { + glugify.slugify("Hello, World!") + |> should.equal("hello-world") +} + +pub fn slugify_transliterates_accents_and_maps_ampersand_test() { + glugify.slugify("Café & Restaurant") + |> should.equal("cafe-and-restaurant") +} + +pub fn slugify_strips_emoji_and_keeps_digits_test() { + glugify.slugify("10 Tips 🚀 for Gleam") + |> should.equal("10-tips-for-gleam") +} + +pub fn slugify_transliterates_cyrillic_and_typographic_punctuation_test() { + glugify.slugify("Привет мир") + |> should.equal("privet-mir") + glugify.slugify("Don’t — “Stop”") + |> should.equal("dont-stop") +} + +pub fn slugify_maps_symbol_only_input_to_words_test() { + glugify.slugify("!@#$%^&*()") + |> should.equal("at-dollar-percent-and") +} + +pub fn slugify_swallows_errors_into_an_empty_string_test() { + glugify.slugify("") + |> should.equal("") + glugify.slugify(" ") + |> should.equal("") +} + +pub fn try_slugify_reports_empty_and_whitespace_only_input_test() { + glugify.try_slugify("") + |> should.equal(Error(errors.EmptyInput)) + glugify.try_slugify(" ") + |> should.equal(Error(errors.EmptyInput)) +} + +pub fn try_slugify_succeeds_on_valid_input_test() { + glugify.try_slugify("My Blog Post Title!") + |> should.equal(Ok("my-blog-post-title")) +} + +pub fn try_slugify_returns_empty_when_every_character_is_stripped_test() { + glugify.try_slugify("🚀🚀") + |> should.equal(Ok("")) +} + +pub fn default_config_values_are_unchanged_test() { + config.default() + |> should.equal( + config.Config( + separator: "-", + lowercase: True, + max_length: option.None, + word_boundary: False, + transliterate: True, + allow_unicode: False, + custom_replacements: [], + preserve_leading_underscore: False, + preserve_trailing_dash: False, + stop_words: [], + trim: True, + locale: locale.Default, + decamelize: False, + decode_html_entities: False, + ignore: [], + ), + ) +} + +pub fn separator_longer_than_ten_characters_is_a_configuration_error_test() { + glugify.slugify_with( + "Hello World", + config.with_separator(config.default(), "this-separator-is-way-too-long"), + ) + |> should.equal( + Error(errors.ConfigurationError("Separator too long (max 10 characters)")), + ) +} + +pub fn separator_of_exactly_ten_characters_is_accepted_test() { + let configuration = config.with_separator(config.default(), "abcdefghij") + config.validate_config(configuration) + |> should.equal(Ok(configuration)) +} + +pub fn custom_separator_and_truncation_at_a_word_boundary_test() { + config.default() + |> config.with_separator("_") + |> config.with_max_length(20) + |> config.with_word_boundary(True) + |> glugify.slugify_with("A Very Long Title That Needs Truncation", _) + |> should.equal(Ok("a_very_long_title")) +} + +pub fn truncation_without_word_boundary_cuts_mid_word_test() { + config.default() + |> config.with_max_length(1) + |> glugify.slugify_with("hello", _) + |> should.equal(Ok("h")) +} + +pub fn max_length_zero_yields_an_empty_slug_test() { + config.default() + |> config.with_max_length(0) + |> glugify.slugify_with("Hello World", _) + |> should.equal(Ok("")) +} + +pub fn negative_max_length_leaves_the_config_unchanged_test() { + config.with_max_length(config.default(), -5).max_length + |> should.equal(option.None) +} + +pub fn seo_preset_truncates_to_sixty_characters_at_a_word_boundary_test() { + glugify.slugify_with( + "A Very Long Production Title That Search Engines Would Rather Not See In Full", + config.seo_preset(), + ) + |> should.equal(Ok("a-very-long-production-title-that-search-engines-would")) +} + +pub fn transliteration_disabled_fails_on_first_non_ascii_grapheme_test() { + config.default() + |> config.with_transliterate(False) + |> glugify.slugify_with("Café", _) + |> should.equal(Error(errors.TransliterationFailed("é"))) +} + +pub fn transliteration_disabled_with_allow_unicode_keeps_accents_test() { + config.default() + |> config.with_transliterate(False) + |> config.with_allow_unicode(True) + |> glugify.slugify_with("Café naïve résumé", _) + |> should.equal(Ok("café-naïve-résumé")) +} + +pub fn lowercase_disabled_keeps_case_test() { + config.default() + |> config.with_lowercase(False) + |> glugify.slugify_with("Hello World", _) + |> should.equal(Ok("Hello-World")) +} + +pub fn custom_replacements_run_before_separators_test() { + config.default() + |> config.with_custom_replacements([ + #("&", " and "), + #("@", " at "), + #("%", " percent "), + ]) + |> glugify.slugify_with("Cats & Dogs @ 100%", _) + |> should.equal(Ok("cats-and-dogs-at-100-percent")) +} + +pub fn stop_words_are_removed_case_insensitively_test() { + config.default() + |> config.with_stop_words(["the", "a", "an", "and", "or"]) + |> glugify.slugify_with("The Quick Brown Fox and the Lazy Dog", _) + |> should.equal(Ok("quick-brown-fox-lazy-dog")) +} + +pub fn stop_words_still_work_with_an_empty_separator_test() { + config.default() + |> config.with_separator("") + |> config.with_stop_words(["the"]) + |> glugify.slugify_with("The Quick Fox", _) + |> should.equal(Ok("quickfox")) +} + +pub fn all_options_combined_test() { + config.default() + |> config.with_separator("_") + |> config.with_lowercase(False) + |> config.with_max_length(50) + |> config.with_word_boundary(True) + |> config.with_transliterate(False) + |> config.with_allow_unicode(True) + |> config.with_custom_replacements([#("&", " and "), #("@", " at ")]) + |> config.with_stop_words(["the", "a"]) + |> glugify.slugify_with("The Amazing @Café & Restaurant", _) + |> should.equal(Ok("Amazing_at_Café_and_Restaurant")) +} + +pub fn german_locale_uses_ae_oe_ue_test() { + config.default() + |> config.with_locale(locale.German) + |> glugify.slugify_with("Über München", _) + |> should.equal(Ok("ueber-muenchen")) +} + +pub fn danish_locale_uses_aa_and_oe_test() { + config.default() + |> config.with_locale(locale.Danish) + |> glugify.slugify_with("København på Ærø", _) + |> should.equal(Ok("koebenhavn-paa-aeroe")) +} + +pub fn decamelize_splits_camel_case_and_acronyms_test() { + config.default() + |> config.with_decamelize(True) + |> glugify.slugify_with("myAwesomeXMLParser", _) + |> should.equal(Ok("my-awesome-xml-parser")) +} + +pub fn html_entities_are_decoded_only_when_enabled_test() { + config.default() + |> config.with_decode_html_entities(True) + |> glugify.slugify_with("Tom & Jerry – Classics", _) + |> should.equal(Ok("tom-and-jerry-classics")) + glugify.slugify("Tom & Jerry") + |> should.equal("tom-and-amp-jerry") +} + +pub fn ignored_graphemes_survive_verbatim_but_are_lowercased_test() { + config.default() + |> config.with_ignore(["#"]) + |> glugify.slugify_with("C# and F# compared", _) + |> should.equal(Ok("c#-and-f#-compared")) + config.default() + |> config.with_ignore(["嗨"]) + |> glugify.slugify_with("嗨 hello", _) + |> should.equal(Ok("嗨-hello")) + config.default() + |> config.with_ignore(["Ü"]) + |> glugify.slugify_with("Ü ber", _) + |> should.equal(Ok("ü-ber")) + config.default() + |> config.with_ignore(["👍🏽"]) + |> glugify.slugify_with("a👍🏽b", _) + |> should.equal(Ok("a👍🏽b")) +} + +pub fn leading_underscore_is_kept_only_when_asked_test() { + config.default() + |> config.with_preserve_leading_underscore(True) + |> glugify.slugify_with("_private notes", _) + |> should.equal(Ok("_private-notes")) + glugify.slugify("_private notes") + |> should.equal("private-notes") +} + +pub fn trailing_dash_is_kept_as_the_configured_separator_test() { + config.default() + |> config.with_preserve_trailing_dash(True) + |> glugify.slugify_with("draft-", _) + |> should.equal(Ok("draft-")) + config.default() + |> config.with_preserve_trailing_dash(True) + |> config.with_separator("_") + |> glugify.slugify_with("draft-", _) + |> should.equal(Ok("draft_")) + glugify.slugify("draft-") + |> should.equal("draft") +} + +pub fn trim_disabled_keeps_a_trailing_separator_test() { + config.default() + |> config.with_trim(False) + |> glugify.slugify_with("hello!", _) + |> should.equal(Ok("hello-")) + config.default() + |> config.with_trim(False) + |> glugify.slugify_with("!hello!", _) + |> should.equal(Ok("hello-")) +} + +pub fn preserve_options_only_act_on_the_exact_edge_characters_test() { + let leading = config.with_preserve_leading_underscore(config.default(), True) + glugify.slugify_with("__x", leading) + |> should.equal(Ok("_x")) + glugify.slugify_with("x_y", leading) + |> should.equal(Ok("x-y")) + let trailing = config.with_preserve_trailing_dash(config.default(), True) + glugify.slugify_with("draft--", trailing) + |> should.equal(Ok("draft-")) + glugify.slugify_with("draft", trailing) + |> should.equal(Ok("draft")) +} + +pub fn word_boundary_truncation_never_ends_on_a_separator_test() { + config.default() + |> config.with_max_length(6) + |> config.with_word_boundary(True) + |> glugify.slugify_with("hello world foo", _) + |> should.equal(Ok("hello")) + config.default() + |> config.with_max_length(5) + |> config.with_word_boundary(True) + |> glugify.slugify_with("helloworld foo", _) + |> should.equal(Ok("hello")) + config.default() + |> config.with_max_length(6) + |> glugify.slugify_with("hello world", _) + |> should.equal(Ok("hello")) +} + +fn entity_config() -> config.Config { + config.with_decode_html_entities(config.default(), True) +} + +pub fn invalid_or_unknown_entities_are_kept_as_text_test() { + glugify.slugify_with("a � b", entity_config()) + |> should.equal(Ok("a-and-xd800-b")) + glugify.slugify_with("a � b", entity_config()) + |> should.equal(Ok("a-and-1114112-b")) + glugify.slugify_with("a &bogus; b", entity_config()) + |> should.equal(Ok("a-and-bogus-b")) + glugify.slugify_with("a & b &", entity_config()) + |> should.equal(Ok("a-and-amp-b-and")) + glugify.slugify_with("café & x 😀 y", entity_config()) + |> should.equal(Ok("cafe-and-x-y")) + glugify.slugify_with( + "a b…c “q” ® © €", + entity_config(), + ) + |> should.equal(Ok("a-b-c-q-r-c-euro")) + glugify.slugify_with("<b> "q" 's'", entity_config()) + |> should.equal(Ok("b-q-s")) + glugify.slugify_with( + "x ° ™ £ ¥ ¢ ‘a’ — y", + entity_config(), + ) + |> should.equal(Ok("x-degrees-tm-pound-yen-cent-a-y")) + glugify.slugify_with("a &abcdefghijkl; b", entity_config()) + |> should.equal(Ok("a-and-abcdefghijkl-b")) + glugify.slugify_with("a &; b", entity_config()) + |> should.equal(Ok("a-and-b")) + glugify.slugify_with("a & b", entity_config()) + |> should.equal(Ok("a-and-b")) + glugify.slugify_with("a &#-5; b", entity_config()) + |> should.equal(Ok("a-and-5-b")) + glugify.slugify_with("5 � 6", entity_config()) + |> should.equal(Ok("5-6")) +} + +pub fn entities_with_unparsable_numbers_are_kept_as_text_test() { + glugify.slugify_with("a &#xzz; b", entity_config()) + |> should.equal(Ok("a-and-xzz-b")) + glugify.slugify_with("a &#abc; b", entity_config()) + |> should.equal(Ok("a-and-abc-b")) +} + +pub fn invisible_codepoints_are_dropped_before_validation_test() { + glugify.slugify("a\u{200D}b c\u{FEFF}d") + |> should.equal("ab-cd") + glugify.try_slugify("\u{200D}") + |> should.equal(Ok("")) + glugify.slugify("שָׁלוֹם") + |> should.equal("shlvm") + glugify.slugify("مَرْحَبًا") + |> should.equal("mrhba") + glugify.slugify("e\u{0301}cole") + |> should.equal("ecole") + glugify.slugify("☃\u{FE0F} x") + |> should.equal("x") + glugify.slugify("x\u{00A0}y\u{3000}z") + |> should.equal("x-y-z") + glugify.slugify("ａｂｃ") + |> should.equal("") + glugify.slugify("test\tit\nnow\r!") + |> should.equal("test-it-now") + config.default() + |> config.with_transliterate(False) + |> glugify.slugify_with("a\u{200D}b", _) + |> should.equal(Error(errors.TransliterationFailed("a\u{200D}"))) +} + +pub fn every_transliteration_table_keeps_its_mappings_test() { + glugify.slugify("Ψυχή θάλασσα Ξ") + |> should.equal("psyxi-thalassa-ks") + glugify.slugify("مرحبا ٣ ۶") + |> should.equal("mrhba-3-6") + glugify.slugify("שלום עולם") + |> should.equal("shlvm-avlm") + glugify.slugify("Ґанок їжак") + |> should.equal("ganok-yizhak") + glugify.slugify("«a» ‹b› ‚c‘ ′d″ a–b a—b") + |> should.equal("a-b-c-d-a-b-a-b") + config.default() + |> config.with_locale(locale.Danish) + |> glugify.slugify_with("Ærø ß", _) + |> should.equal(Ok("aeroe-ss")) + config.default() + |> config.with_locale(locale.German) + |> glugify.slugify_with("Straße ÖÄ", _) + |> should.equal(Ok("strasse-oeae")) + config.default() + |> config.with_locale(locale.Norwegian) + |> glugify.slugify_with("Ærø", _) + |> should.equal(Ok("aeroe")) + config.default() + |> config.with_locale(locale.Swedish) + |> glugify.slugify_with("Ærø åäö", _) + |> should.equal(Ok("aero-aao")) + config.default() + |> config.with_locale(locale.Turkish) + |> glugify.slugify_with("Işık ğüş", _) + |> should.equal(Ok("isik-gus")) +} + +pub fn decamelize_keeps_digits_attached_and_leaves_lowercase_alone_test() { + let camel = config.with_decamelize(config.default(), True) + glugify.slugify_with("version2Beta HTMLParser5", camel) + |> should.equal(Ok("version2-beta-html-parser5")) + glugify.slugify_with("already lower", camel) + |> should.equal(Ok("already-lower")) + glugify.slugify_with("ABc a1B ÉcoleDuMonde", camel) + |> should.equal(Ok("a-bc-a1-b-ecole-du-monde")) + glugify.slugify_with("x👍🏽Y", camel) + |> should.equal(Ok("xy")) +} + +pub fn multi_character_and_empty_separators_test() { + config.default() + |> config.with_separator("--") + |> glugify.slugify_with("a b--c", _) + |> should.equal(Ok("a--b--c")) + config.default() + |> config.with_separator("") + |> glugify.slugify_with("!a b!", _) + |> should.equal(Ok("ab")) + config.default() + |> config.with_separator("") + |> config.with_max_length(7) + |> glugify.slugify_with("hello world", _) + |> should.equal(Ok("hellowo")) + config.default() + |> config.with_separator("") + |> config.with_max_length(7) + |> config.with_word_boundary(True) + |> glugify.slugify_with("hello world", _) + |> should.equal(Ok("hellowo")) + config.default() + |> config.with_max_length(6) + |> config.with_word_boundary(True) + |> glugify.slugify_with("ab-cd-ef", _) + |> should.equal(Ok("ab-cd")) + config.default() + |> config.with_max_length(3) + |> glugify.slugify_with("ab-cd-ef", _) + |> should.equal(Ok("ab")) + config.default() + |> config.with_separator("_") + |> config.with_stop_words(["the"]) + |> glugify.slugify_with("The A Team", _) + |> should.equal(Ok("a_team")) +} + +pub fn stop_words_that_consume_everything_give_an_empty_slug_test() { + config.default() + |> config.with_stop_words(["the", "a"]) + |> glugify.slugify_with("the a", _) + |> should.equal(Ok("")) +} + +pub fn slugger_numbers_duplicates_from_one_test() { + let #(s, first) = slugger.slug(slugger.new(), "Hello World") + let #(s, second) = slugger.slug(s, "Hello World") + let #(_, third) = slugger.slug(s, "Hello World") + #(first, second, third) + |> should.equal(#("hello-world", "hello-world-1", "hello-world-2")) +} + +pub fn slugger_never_collides_with_genuinely_suffixed_input_test() { + let #(s, first) = slugger.slug(slugger.new(), "foo-1") + let #(s, second) = slugger.slug(s, "foo") + let #(_, third) = slugger.slug(s, "foo") + #(first, second, third) + |> should.equal(#("foo-1", "foo", "foo-2")) +} + +pub fn slugger_returns_empty_slugs_without_numbering_test() { + let #(s, first) = slugger.slug(slugger.new(), "!!!") + let #(_, second) = slugger.slug(s, "!!!") + #(first, second) + |> should.equal(#("", "")) +} + +pub fn slug_with_applies_a_config_and_still_numbers_with_a_hyphen_test() { + let underscore = config.with_separator(config.default(), "_") + let assert Ok(#(s, first)) = + slugger.slug_with(slugger.new(), "My Post", underscore) + let assert Ok(#(_, second)) = slugger.slug_with(s, "My Post", underscore) + #(first, second) + |> should.equal(#("my_post", "my_post-1")) + slugger.slug_with( + slugger.new(), + "My Post", + config.with_separator(config.default(), "this-separator-is-way-too-long"), + ) + |> should.equal( + Error(errors.ConfigurationError("Separator too long (max 10 characters)")), + ) +} + +pub fn to_anchor_follows_github_slugger_rules_test() { + anchor.to_anchor("Hello, World!") + |> should.equal("hello-world") + anchor.to_anchor("Hello, _World_!") + |> should.equal("hello-_world_") + anchor.to_anchor("a b") + |> should.equal("a--b") + anchor.to_anchor(":ok_hand: Single") + |> should.equal("ok_hand-single") + anchor.to_anchor("I ♥ unicode") + |> should.equal("i--unicode") +} + +pub fn to_anchor_maintaining_case_keeps_case_test() { + anchor.to_anchor_maintaining_case("Hello, World!") + |> should.equal("Hello-World") + anchor.to_anchor_maintaining_case("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃") + |> should.equal("Ⓐ-🄰-١٢-Ⅻ-a_b-‿--xy-") +} + +pub fn to_anchor_keeps_letters_marks_digits_and_circled_letters_test() { + anchor.to_anchor("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃") + |> should.equal("ⓐ-🄰-١٢-ⅻ-a_b-‿--xy-") + anchor.to_anchor(" a b ") + |> should.equal("--a---b--") + anchor.to_anchor("tab\there") + |> should.equal("tabhere") +} + +pub fn anchorer_numbers_duplicate_headings_test() { + let #(a, first) = anchor.anchor(anchor.new(), "Intro") + let #(_, second) = anchor.anchor(a, "Intro") + #(first, second) + |> should.equal(#("intro", "intro-1")) + let #(b, third) = anchor.anchor_maintaining_case(anchor.new(), "Intro") + let #(_, fourth) = anchor.anchor_maintaining_case(b, "Intro") + #(third, fourth) + |> should.equal(#("Intro", "Intro-1")) +} + +pub fn unicode_transliterate_text_strips_unmappable_characters_test() { + unicode.transliterate_text("Résumé 🚀") + |> should.equal(Ok("Resume ")) + unicode.transliterate_text_with("Über München", locale.German, []) + |> should.equal(Ok("Ueber Muenchen")) + unicode.transliterate_text("e\u{0301}") + |> should.equal(Ok("e")) +} + +pub fn unicode_validate_ascii_only_and_validate_ascii_or_unicode_test() { + unicode.validate_ascii_only("hello world") + |> should.equal(Ok("hello world")) + unicode.validate_ascii_only("café") + |> should.equal(Error(errors.TransliterationFailed("é"))) + unicode.validate_ascii_only("e\u{0301}") + |> should.equal(Error(errors.TransliterationFailed("e\u{0301}"))) + unicode.validate_ascii_or_unicode("café", False, []) + |> should.equal(Error(errors.TransliterationFailed("é"))) + unicode.validate_ascii_or_unicode("café", True, []) + |> should.equal(Ok("café")) + unicode.validate_ascii_or_unicode("café", False, ["é"]) + |> should.equal(Ok("café")) +} + +pub fn transformations_compose_left_to_right_and_stop_on_first_error_test() { + let pipeline = + transformations.compose([ + transformations.normalize_unicode(), + transformations.transliterate_to_ascii(), + ]) + pipeline("café") + |> should.equal(Ok("cafe")) + transformations.compose([])("x") + |> should.equal(Ok("x")) + transformations.compose([ + transformations.transliterate_to_ascii(), + fn(_) { Error(errors.EmptyInput) }, + ])("café") + |> should.equal(Error(errors.EmptyInput)) +} diff --git a/test/public_api_test.gleam b/test/public_api_test.gleam new file mode 100644 index 0000000..14d13d7 --- /dev/null +++ b/test/public_api_test.gleam @@ -0,0 +1,147 @@ +import gleam/option.{type Option} +import gleeunit/should +import glugify +import glugify/anchor.{type Anchorer} +import glugify/config.{type Config} +import glugify/errors.{type SlugifyError} +import glugify/locale.{type Locale} +import glugify/slugger.{type Slugger} +import glugify/transformations.{type Transformation} +import glugify/unicode + +type Surface { + Surface( + slugify: fn(String) -> String, + try_slugify: fn(String) -> Result(String, SlugifyError), + slugify_with: fn(String, Config) -> Result(String, SlugifyError), + default: fn() -> Config, + seo_preset: fn() -> Config, + validate_config: fn(Config) -> Result(Config, String), + with_separator: fn(Config, String) -> Config, + with_lowercase: fn(Config, Bool) -> Config, + with_max_length: fn(Config, Int) -> Config, + with_word_boundary: fn(Config, Bool) -> Config, + with_transliterate: fn(Config, Bool) -> Config, + with_allow_unicode: fn(Config, Bool) -> Config, + with_custom_replacements: fn(Config, List(#(String, String))) -> Config, + with_stop_words: fn(Config, List(String)) -> Config, + with_preserve_leading_underscore: fn(Config, Bool) -> Config, + with_preserve_trailing_dash: fn(Config, Bool) -> Config, + with_locale: fn(Config, Locale) -> Config, + with_decamelize: fn(Config, Bool) -> Config, + with_decode_html_entities: fn(Config, Bool) -> Config, + with_ignore: fn(Config, List(String)) -> Config, + with_trim: fn(Config, Bool) -> Config, + new_slugger: fn() -> Slugger, + slug: fn(Slugger, String) -> #(Slugger, String), + slug_with: fn(Slugger, String, Config) -> + Result(#(Slugger, String), SlugifyError), + to_anchor: fn(String) -> String, + to_anchor_maintaining_case: fn(String) -> String, + new_anchorer: fn() -> Anchorer, + anchor: fn(Anchorer, String) -> #(Anchorer, String), + anchor_maintaining_case: fn(Anchorer, String) -> #(Anchorer, String), + transliterate_text: fn(String) -> Result(String, SlugifyError), + transliterate_text_with: fn(String, Locale, List(String)) -> + Result(String, SlugifyError), + validate_ascii_only: fn(String) -> Result(String, SlugifyError), + validate_ascii_or_unicode: fn(String, Bool, List(String)) -> + Result(String, SlugifyError), + compose: fn(List(Transformation)) -> Transformation, + normalize_unicode: fn() -> Transformation, + transliterate_to_ascii: fn() -> Transformation, + max_length_of: fn(Config) -> Option(Int), + locales: List(Locale), + slugify_errors: List(SlugifyError), + ) +} + +fn surface() -> Surface { + Surface( + slugify: glugify.slugify, + try_slugify: glugify.try_slugify, + slugify_with: glugify.slugify_with, + default: config.default, + seo_preset: config.seo_preset, + validate_config: config.validate_config, + with_separator: config.with_separator, + with_lowercase: config.with_lowercase, + with_max_length: config.with_max_length, + with_word_boundary: config.with_word_boundary, + with_transliterate: config.with_transliterate, + with_allow_unicode: config.with_allow_unicode, + with_custom_replacements: config.with_custom_replacements, + with_stop_words: config.with_stop_words, + with_preserve_leading_underscore: config.with_preserve_leading_underscore, + with_preserve_trailing_dash: config.with_preserve_trailing_dash, + with_locale: config.with_locale, + with_decamelize: config.with_decamelize, + with_decode_html_entities: config.with_decode_html_entities, + with_ignore: config.with_ignore, + with_trim: config.with_trim, + new_slugger: slugger.new, + slug: slugger.slug, + slug_with: slugger.slug_with, + to_anchor: anchor.to_anchor, + to_anchor_maintaining_case: anchor.to_anchor_maintaining_case, + new_anchorer: anchor.new, + anchor: anchor.anchor, + anchor_maintaining_case: anchor.anchor_maintaining_case, + transliterate_text: unicode.transliterate_text, + transliterate_text_with: unicode.transliterate_text_with, + validate_ascii_only: unicode.validate_ascii_only, + validate_ascii_or_unicode: unicode.validate_ascii_or_unicode, + compose: transformations.compose, + normalize_unicode: transformations.normalize_unicode, + transliterate_to_ascii: transformations.transliterate_to_ascii, + max_length_of: fn(configuration: Config) { configuration.max_length }, + locales: [ + locale.Default, + locale.German, + locale.Danish, + locale.Norwegian, + locale.Swedish, + locale.Turkish, + ], + slugify_errors: [ + errors.EmptyInput, + errors.TransliterationFailed("x"), + errors.ConfigurationError("x"), + ], + ) +} + +pub fn every_exported_name_in_the_feature_still_exists_test() { + let api = surface() + api.slugify("Hello, World!") + |> should.equal("hello-world") + api.max_length_of(api.seo_preset()) + |> should.equal(option.Some(60)) + api.locales + |> should.equal([ + locale.Default, + locale.German, + locale.Danish, + locale.Norwegian, + locale.Swedish, + locale.Turkish, + ]) + api.slugify_errors + |> should.equal([ + errors.EmptyInput, + errors.TransliterationFailed("x"), + errors.ConfigurationError("x"), + ]) +} + +pub fn the_javascript_ffi_default_export_backs_the_anchor_functions_test() { + anchor.to_anchor_maintaining_case("Ⓐ 🄰 ١٢ Ⅻ a_b ‿ ・ x—y ☃") + |> should.equal("Ⓐ-🄰-١٢-Ⅻ-a_b-‿--xy-") + anchor.to_anchor(" a b ") + |> should.equal("--a---b--") +} + +pub fn anchors_keep_common_scripts_on_every_host_unicode_table_test() { + anchor.to_anchor("Ψυχή жизнь 漢字 مرحبا") + |> should.equal("ψυχή-жизнь-漢字-مرحبا") +}