From effac0a4856d857f26a9eee867c6f639453d0c57 Mon Sep 17 00:00:00 2001 From: FletcherDares <52580867+FletcherDares@users.noreply.github.com> Date: Tue, 26 May 2026 00:52:41 -0400 Subject: [PATCH] Improve performance of `String#scrub` by skipping ASCII runs with `search_nonascii` (#16359) Improve performance of String#scrub by skipping ascii characters after finding a multi byte character using search_nonascii. Create benchmark for String#scrub to show performance difference made by including the new ascii skips. --- benchmark/string_scrub.yml | 48 ++++++++++++++++++++++++++++++++++++++ string.c | 12 ++++++++++ 2 files changed, 60 insertions(+) create mode 100644 benchmark/string_scrub.yml diff --git a/benchmark/string_scrub.yml b/benchmark/string_scrub.yml new file mode 100644 index 0000000000..4b5faaad8e --- /dev/null +++ b/benchmark/string_scrub.yml @@ -0,0 +1,48 @@ +prelude: | + + STRING_SIZE = 1024 + def duplicate_to_length(str, target_length) + return "" if target_length <= 0 + return str[0, target_length] if str.length >= target_length + + (str * ((target_length / str.length) + 1))[0, target_length] + end + base = "Hello \u{1f600} world! \u{00e9}\u{00f1}" + padding = duplicate_to_length(base, STRING_SIZE) + + valid_utf8 = (padding.b + "OK".b).force_encoding("UTF-8") + valid_utf8.valid_encoding? + unknown_but_valid_utf8 = valid_utf8.dup.b.force_encoding("UTF-8") + invalid_utf8 = (padding.b + "\x80\xFF".b).force_encoding("UTF-8") + invalid_utf8.valid_encoding? + unknown_but_invalid_utf8 = (padding.b + "\x80\xFF".b).force_encoding("UTF-8") + + worst_case_utf8 = duplicate_to_length("\u{1f600}\u{00e9}\u{00f1}", STRING_SIZE).b.force_encoding("UTF-8") + + unknown_but_valid_utf8_worst_case = worst_case_utf8.dup.b.force_encoding("UTF-8") + unknown_but_invalid_utf8_worst_case = (worst_case_utf8.b + "\x80\xFF".b).force_encoding("UTF-8") + +benchmark: + scrub_known_valid: | + string = valid_utf8.dup + string.scrub! + + scrub_known_invalid: | + string = invalid_utf8.dup + string.scrub! + + scrub_unknown_but_valid_coderange: | + string = unknown_but_valid_utf8.dup + string.scrub! + + scrub_unknown_and_invalid_coderange: | + string = unknown_but_invalid_utf8.dup + string.scrub! + + scrub_unknown_but_valid_coderange_worst_case: | + string = unknown_but_valid_utf8_worst_case.dup + string.scrub! + + scrub_unknown_and_invalid_coderange_worst_case: | + string = unknown_but_invalid_utf8_worst_case.dup + string.scrub! \ No newline at end of file diff --git a/string.c b/string.c index a59340adfd..6865b0d8e6 100644 --- a/string.c +++ b/string.c @@ -11850,6 +11850,18 @@ enc_str_scrub(rb_encoding *enc, VALUE str, VALUE repl, int cr) else if (MBCLEN_CHARFOUND_P(ret)) { cr = ENC_CODERANGE_VALID; p += MBCLEN_CHARFOUND_LEN(ret); + /* + * After a valid multibyte character, skip the following ASCII run. + * If the next byte is already non-ASCII, search_nonascii would only + * rediscover p after its word-at-a-time setup. + */ + if (p < e && ISASCII(*p)) { + p = search_nonascii(p, e); + if (!p) { + p = e; + break; + } + } } else if (MBCLEN_INVALID_P(ret)) { /*