From 550a0b4286858d6d32eb916941f4a9a7d2493269 Mon Sep 17 00:00:00 2001 From: sarankumar1325 <120080287+sarankumar1325@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:00:16 +0530 Subject: [PATCH] cut: trim Unicode blanks with -w=trimmed In a UTF-8 locale the whitespace matcher splits fields on Unicode blanks such as U+3000, but -w=trimmed only stripped SPACE and TAB from the ends of a line. A leading Unicode blank then emptied field 1 and shifted every word, a trailing one added an empty last field, and -s kept one-word lines. Trim with the same matcher that splits the fields, and update the help text, which still said Space/Tab only. Fixes #15819 --- src/uu/cut/locales/en-US.ftl | 2 +- src/uu/cut/src/cut.rs | 24 ++++++++++++++++++------ tests/by-util/test_cut.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/uu/cut/locales/en-US.ftl b/src/uu/cut/locales/en-US.ftl index 6e63feb2638..4d40825784d 100644 --- a/src/uu/cut/locales/en-US.ftl +++ b/src/uu/cut/locales/en-US.ftl @@ -20,7 +20,7 @@ cut-after-help = If no FILE is specified, reads from stdin. Use '-' as a FILE ar cut-help-bytes = extract bytes listed in LIST cut-help-characters = extract characters listed in LIST cut-help-delimiter = use DELIM as field separator (default: Tab) -cut-help-whitespace-delimited = use any whitespace (Space/Tab) as delimiter; ignore leading and trailing blanks with 'trimmed' +cut-help-whitespace-delimited = use any run of blanks (Space/Tab, plus Unicode spaces in a UTF-8 locale) as delimiter; ignore leading and trailing blanks with 'trimmed' cut-help-fields = extract fields listed in LIST cut-help-fields-merged = like -f, but merge adjacent delimiters; the delimiter defaults to whitespace and the output delimiter to a space cut-help-complement = invert selection: print all columns except the specified ones diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 9680ece4a02..9e8a2d6ab79 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -780,19 +780,31 @@ fn cut_fields_whitespace_trimmed( newline_char: u8, out_delim: &[u8], ) -> UResult<()> { - let is_blank = |&b: &u8| b == b' ' || b == b'\t'; let matcher = WhitespaceMatcher {}; let mut buf_in = BufReader::new(reader); // Trimming is the only difference from the plain whitespace path: once the // outer blanks are gone, the remaining blank runs are ordinary delimiters. + // The ends are trimmed with the same matcher that splits the fields, so + // in a UTF-8 locale a Unicode blank at either end is trimmed rather than + // turned into a leading or trailing delimiter. buf_in .for_byte_record(newline_char, |line| { - let start = line.iter().position(|b| !is_blank(b)).unwrap_or(line.len()); - let end = line - .iter() - .rposition(|b| !is_blank(b)) - .map_or(start, |p| p + 1); + let start = match matcher.next_match(line) { + Some((0, run_end)) => run_end, + _ => 0, + }; + let rest = &line[start..]; + let mut end = rest.len(); + let mut pos = 0; + while let Some((run_start, run_end)) = matcher.next_match(&rest[pos..]) { + if pos + run_end == rest.len() { + end = pos + run_start; + break; + } + pos += run_end; + } + let end = start + end; write_fields_line( &line[start..end], diff --git a/tests/by-util/test_cut.rs b/tests/by-util/test_cut.rs index 294863baa73..a8420c179d3 100644 --- a/tests/by-util/test_cut.rs +++ b/tests/by-util/test_cut.rs @@ -834,6 +834,35 @@ fn test_whitespace_delimited_trimmed_zero_terminated() { .stdout_only_bytes(&b"amber\0"[..]); } +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] +#[cfg(target_os = "linux")] +fn test_whitespace_delimited_trimmed_unicode_blanks() { + // In a UTF-8 locale the field splitter treats Unicode blanks such as + // U+3000 (e3 80 80) as delimiters, so trimming must strip them too, or a + // leading one empties field 1 and a trailing one adds an empty last field. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-w=trimmed", "-f1"]) + .pipe_in(&b"\xe3\x80\x80shutdown now\n"[..]) + .succeeds() + .stdout_only("shutdown\n"); + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-w=trimmed", "-f2-", "--output-delimiter=|"]) + .pipe_in(&b" \xe3\x80\x80red\xe2\x80\x83blue \xe3\x80\x80\n"[..]) + .succeeds() + .stdout_only("blue\n"); + // A one-word line padded with Unicode blanks has no delimiter left after + // trimming, so -s drops it. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-s", "-w=trimmed", "-f1"]) + .pipe_in(&b"\xe3\x80\x80alone\xe3\x80\x80\nnot alone\n"[..]) + .succeeds() + .stdout_only("not\n"); +} + #[test] #[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] #[cfg(target_os = "linux")]