From e3ea84764a0759f0f675602cd626e168f13ab559 Mon Sep 17 00:00:00 2001 From: preciz Date: Fri, 2 Oct 2026 11:23:59 +0200 Subject: [PATCH 1/5] Optimize String.chunk with run scanning and sub-binary slicing Scan matching and nonmatching runs directly, slicing chunks at their boundaries instead of constructing binaries for every codepoint. Reuse the original input for uniform strings and keep chunk accumulation tail recursive. Scan ASCII prefixes in seven-byte groups with UTF-8 fallback. Share printable codepoint classification between chunk/2 and printable?/2, and inline the character-limit decrement in printable?/2. Assisted-by: Gemini:3.8-Flash Assisted-by: Codex:GPT-6 --- lib/elixir/lib/string.ex | 109 +++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 50e959881d..70decd3e0c 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -309,6 +309,24 @@ defmodule String do @conditional_mappings [:greek, :turkic] + # ::utf8 already rejects surrogates and code points above 0x10FFFF. + defguardp is_printable_non_ascii_codepoint(codepoint) + when codepoint >= 0xA0 and codepoint not in 0xFFFE..0xFFFF + + # Checks one Unicode code point decoded by ::utf8, including printable control characters. + defguardp is_printable_codepoint(codepoint) + when codepoint in 0x20..0x7F or + codepoint in ?\a..?\r or codepoint == ?\e or + is_printable_non_ascii_codepoint(codepoint) + + # ::56 packs seven bytes into an integer; the mask checks each byte's high bit. + defguardp is_ascii(bytes) when Bitwise.band(bytes, 0x80808080808080) == 0 + + # The ::56 fast path accepts ASCII bytes >= 0x20; tabs and newlines use ::utf8. + defguardp is_printable_ascii_bytes(bytes) + when is_ascii(bytes) and + Bitwise.band(bytes - 0x20202020202020, 0x80808080808080) == 0 + @doc """ Checks if a string contains only printable characters up to `character_limit`. @@ -342,22 +360,8 @@ defmodule String do defp recur_printable?(<<_::binary>>, 0), do: true defp recur_printable?(<<>>, _character_limit), do: true - for char <- 0x20..0x7E do - defp recur_printable?(<>, character_limit) do - recur_printable?(rest, decrement(character_limit)) - end - end - - for char <- [?\n, ?\r, ?\t, ?\v, ?\b, ?\f, ?\e, ?\d, ?\a] do - defp recur_printable?(<>, character_limit) do - recur_printable?(rest, decrement(character_limit)) - end - end - - defp recur_printable?(<>, character_limit) - when char in 0xA0..0xD7FF - when char in 0xE000..0xFFFD - when char in 0x10000..0x10FFFF do + defp recur_printable?(<>, character_limit) + when is_printable_codepoint(codepoint) do recur_printable?(rest, decrement(character_limit)) end @@ -365,8 +369,10 @@ defmodule String do false end - defp decrement(:infinity), do: :infinity - defp decrement(character_limit), do: character_limit - 1 + @compile {:inline, decrement: 1} + defp decrement(character_limit) do + if character_limit == :infinity, do: :infinity, else: character_limit - 1 + end @doc ~S""" Divides a string into substrings at each Unicode whitespace @@ -2151,29 +2157,66 @@ defmodule String do def chunk("", _), do: [] def chunk(string, trait) when is_binary(string) and trait in [:valid, :printable] do - {cp, _} = next_codepoint(string) - pred_fn = make_chunk_pred(trait) - do_chunk(string, pred_fn.(cp), pred_fn) + rest = + case trait do + :valid -> drop_valid_utf8_prefix(string) + :printable -> drop_printable_prefix(string) + end + + if byte_size(rest) == byte_size(string) do + do_chunk(string, false, trait, []) + else + add_chunk(string, rest, true, trait, []) + end end - defp do_chunk(string, flag, pred_fn), do: do_chunk(string, [], <<>>, flag, pred_fn) + defp do_chunk(string, matches?, :valid, acc) do + rest = if matches?, do: drop_valid_utf8_prefix(string), else: drop_invalid_utf8_prefix(string) - defp do_chunk(<<>>, acc, <<>>, _, _), do: Enum.reverse(acc) + add_chunk(string, rest, matches?, :valid, acc) + end - defp do_chunk(<<>>, acc, chunk, _, _), do: Enum.reverse(acc, [chunk]) + defp do_chunk(string, matches?, :printable, acc) do + rest = if matches?, do: drop_printable_prefix(string), else: drop_unprintable_prefix(string) - defp do_chunk(string, acc, chunk, flag, pred_fn) do - {cp, rest} = next_codepoint(string) + add_chunk(string, rest, matches?, :printable, acc) + end - if pred_fn.(cp) != flag do - do_chunk(rest, [chunk | acc], cp, not flag, pred_fn) - else - do_chunk(rest, acc, chunk <> cp, flag, pred_fn) - end + defp add_chunk(string, <<>>, _matches?, _trait, acc) do + :lists.reverse(acc, [string]) + end + + defp add_chunk(string, rest, matches?, trait, acc) do + chunk = binary_part(string, 0, byte_size(string) - byte_size(rest)) + do_chunk(rest, not matches?, trait, [chunk | acc]) + end + + defp drop_valid_utf8_prefix(<>) when is_ascii(bytes) do + drop_valid_utf8_prefix(rest) + end + + defp drop_valid_utf8_prefix(<<_::utf8, rest::binary>>), do: drop_valid_utf8_prefix(rest) + defp drop_valid_utf8_prefix(rest), do: rest + + defp drop_invalid_utf8_prefix(<<_::utf8, _::binary>> = rest), do: rest + defp drop_invalid_utf8_prefix(<<_, rest::binary>>), do: drop_invalid_utf8_prefix(rest) + defp drop_invalid_utf8_prefix(<<>>), do: <<>> + + defp drop_printable_prefix(<>) when is_printable_ascii_bytes(bytes), + do: drop_printable_prefix(rest) + + defp drop_printable_prefix(<>) + when is_printable_codepoint(codepoint), + do: drop_printable_prefix(rest) + + defp drop_printable_prefix(rest), do: rest + + defp drop_unprintable_prefix(<> = rest) do + if is_printable_codepoint(codepoint), do: rest, else: drop_unprintable_prefix(tail) end - defp make_chunk_pred(:valid), do: &valid?/1 - defp make_chunk_pred(:printable), do: &printable?/1 + defp drop_unprintable_prefix(<<_, rest::binary>>), do: drop_unprintable_prefix(rest) + defp drop_unprintable_prefix(<<>>), do: <<>> @doc ~S""" Returns Unicode graphemes in the string as per Extended Grapheme From 1e45c6864afa016cf738236b17285e7b2ad3fa54 Mon Sep 17 00:00:00 2001 From: preciz Date: Mon, 5 Oct 2026 23:17:43 +0200 Subject: [PATCH 2/5] Reduce LOC --- lib/elixir/lib/string.ex | 90 ++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 70decd3e0c..a3793d00dc 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -309,15 +309,11 @@ defmodule String do @conditional_mappings [:greek, :turkic] - # ::utf8 already rejects surrogates and code points above 0x10FFFF. - defguardp is_printable_non_ascii_codepoint(codepoint) - when codepoint >= 0xA0 and codepoint not in 0xFFFE..0xFFFF - # Checks one Unicode code point decoded by ::utf8, including printable control characters. defguardp is_printable_codepoint(codepoint) when codepoint in 0x20..0x7F or codepoint in ?\a..?\r or codepoint == ?\e or - is_printable_non_ascii_codepoint(codepoint) + codepoint >= 0xA0 and codepoint not in 0xFFFE..0xFFFF # ::56 packs seven bytes into an integer; the mask checks each byte's high bit. defguardp is_ascii(bytes) when Bitwise.band(bytes, 0x80808080808080) == 0 @@ -354,7 +350,11 @@ defmodule String do when is_binary(string) and (character_limit == :infinity or (is_integer(character_limit) and character_limit >= 0)) do - recur_printable?(string, character_limit) + if character_limit == :infinity do + drop_printable(string, true) == "" + else + recur_printable?(string, character_limit) + end end defp recur_printable?(<<_::binary>>, 0), do: true @@ -362,18 +362,13 @@ defmodule String do defp recur_printable?(<>, character_limit) when is_printable_codepoint(codepoint) do - recur_printable?(rest, decrement(character_limit)) + recur_printable?(rest, character_limit - 1) end defp recur_printable?(_string, _character_limit) do false end - @compile {:inline, decrement: 1} - defp decrement(character_limit) do - if character_limit == :infinity, do: :infinity, else: character_limit - 1 - end - @doc ~S""" Divides a string into substrings at each Unicode whitespace occurrence with leading and trailing whitespace ignored. @@ -2157,66 +2152,41 @@ defmodule String do def chunk("", _), do: [] def chunk(string, trait) when is_binary(string) and trait in [:valid, :printable] do + do_chunk(string, true, trait, []) + end + + defp do_chunk(string, matches?, trait, acc) do rest = case trait do - :valid -> drop_valid_utf8_prefix(string) - :printable -> drop_printable_prefix(string) + :valid -> drop_valid(string, matches?) + :printable -> drop_printable(string, matches?) end - if byte_size(rest) == byte_size(string) do - do_chunk(string, false, trait, []) - else - add_chunk(string, rest, true, trait, []) - end - end - - defp do_chunk(string, matches?, :valid, acc) do - rest = if matches?, do: drop_valid_utf8_prefix(string), else: drop_invalid_utf8_prefix(string) + size = byte_size(string) - byte_size(rest) + acc = if size == 0, do: acc, else: [binary_part(string, 0, size) | acc] - add_chunk(string, rest, matches?, :valid, acc) + if rest == "", do: :lists.reverse(acc), else: do_chunk(rest, not matches?, trait, acc) end - defp do_chunk(string, matches?, :printable, acc) do - rest = if matches?, do: drop_printable_prefix(string), else: drop_unprintable_prefix(string) - - add_chunk(string, rest, matches?, :printable, acc) - end - - defp add_chunk(string, <<>>, _matches?, _trait, acc) do - :lists.reverse(acc, [string]) - end - - defp add_chunk(string, rest, matches?, trait, acc) do - chunk = binary_part(string, 0, byte_size(string) - byte_size(rest)) - do_chunk(rest, not matches?, trait, [chunk | acc]) - end - - defp drop_valid_utf8_prefix(<>) when is_ascii(bytes) do - drop_valid_utf8_prefix(rest) - end - - defp drop_valid_utf8_prefix(<<_::utf8, rest::binary>>), do: drop_valid_utf8_prefix(rest) - defp drop_valid_utf8_prefix(rest), do: rest - - defp drop_invalid_utf8_prefix(<<_::utf8, _::binary>> = rest), do: rest - defp drop_invalid_utf8_prefix(<<_, rest::binary>>), do: drop_invalid_utf8_prefix(rest) - defp drop_invalid_utf8_prefix(<<>>), do: <<>> - - defp drop_printable_prefix(<>) when is_printable_ascii_bytes(bytes), - do: drop_printable_prefix(rest) + defp drop_valid(<>, true) when is_ascii(bytes), + do: drop_valid(rest, true) - defp drop_printable_prefix(<>) - when is_printable_codepoint(codepoint), - do: drop_printable_prefix(rest) + defp drop_valid(<<_::utf8, rest::binary>>, true), do: drop_valid(rest, true) + defp drop_valid(<<_::utf8, _::binary>> = rest, false), do: rest + defp drop_valid(<<_, rest::binary>>, false), do: drop_valid(rest, false) + defp drop_valid(rest, _matches?), do: rest - defp drop_printable_prefix(rest), do: rest + defp drop_printable(<>, true) when is_printable_ascii_bytes(bytes), + do: drop_printable(rest, true) - defp drop_unprintable_prefix(<> = rest) do - if is_printable_codepoint(codepoint), do: rest, else: drop_unprintable_prefix(tail) + defp drop_printable(<> = string, matches?) do + if is_printable_codepoint(codepoint) == matches?, + do: drop_printable(rest, matches?), + else: string end - defp drop_unprintable_prefix(<<_, rest::binary>>), do: drop_unprintable_prefix(rest) - defp drop_unprintable_prefix(<<>>), do: <<>> + defp drop_printable(<<_, rest::binary>>, false), do: drop_printable(rest, false) + defp drop_printable(rest, _matches?), do: rest @doc ~S""" Returns Unicode graphemes in the string as per Extended Grapheme From c3c88f57abbecbe20e45f93fef22f0be9cf7b42d Mon Sep 17 00:00:00 2001 From: preciz Date: Tue, 6 Oct 2026 00:16:28 +0200 Subject: [PATCH 3/5] Run formatter --- lib/elixir/lib/string.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index a3793d00dc..e9bd819385 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -313,7 +313,7 @@ defmodule String do defguardp is_printable_codepoint(codepoint) when codepoint in 0x20..0x7F or codepoint in ?\a..?\r or codepoint == ?\e or - codepoint >= 0xA0 and codepoint not in 0xFFFE..0xFFFF + (codepoint >= 0xA0 and codepoint not in 0xFFFE..0xFFFF) # ::56 packs seven bytes into an integer; the mask checks each byte's high bit. defguardp is_ascii(bytes) when Bitwise.band(bytes, 0x80808080808080) == 0 From ecf7391f2d75282e0a7487fda5e8e86f0f9d71bd Mon Sep 17 00:00:00 2001 From: preciz Date: Tue, 6 Oct 2026 00:19:28 +0200 Subject: [PATCH 4/5] Better syntax --- lib/elixir/lib/string.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index e9bd819385..9d4aa41449 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -347,14 +347,14 @@ defmodule String do @spec printable?(t, 0) :: true @spec printable?(t, pos_integer | :infinity) :: boolean def printable?(string, character_limit \\ :infinity) - when is_binary(string) and - (character_limit == :infinity or - (is_integer(character_limit) and character_limit >= 0)) do - if character_limit == :infinity do - drop_printable(string, true) == "" - else - recur_printable?(string, character_limit) - end + + def printable?(string, :infinity) when is_binary(string) do + drop_printable(string, true) == "" + end + + def printable?(string, character_limit) + when is_binary(string) and is_integer(character_limit) and character_limit >= 0 do + recur_printable?(string, character_limit) end defp recur_printable?(<<_::binary>>, 0), do: true From 5ca03e4f2cc0720f5316fbb863882c3b7afca7f3 Mon Sep 17 00:00:00 2001 From: preciz Date: Tue, 6 Oct 2026 00:47:59 +0200 Subject: [PATCH 5/5] Syntax --- lib/elixir/lib/string.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index 9d4aa41449..a7e2265030 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -2180,9 +2180,11 @@ defmodule String do do: drop_printable(rest, true) defp drop_printable(<> = string, matches?) do - if is_printable_codepoint(codepoint) == matches?, - do: drop_printable(rest, matches?), - else: string + if is_printable_codepoint(codepoint) == matches? do + drop_printable(rest, matches?) + else + string + end end defp drop_printable(<<_, rest::binary>>, false), do: drop_printable(rest, false)