expand/unexpand: fix display width of 3- and 4-byte UTF-8 characters

src/uu/expand/Cargo.toml

@@ -23,7 +23,6 @@ clap = { workspace = true }
 fluent = { workspace = true }
 rustix = { workspace = true }
 thiserror = { workspace = true }
-unicode-width = { workspace = true }
 uucore = { workspace = true }
 
 [dev-dependencies]

src/uu/expand/src/expand.rs

@@ -13,7 +13,7 @@ use std::num::IntErrorKind;
 use std::path::Path;
 use std::str::from_utf8;
 use thiserror::Error;
-use unicode_width::UnicodeWidthChar;
+use uucore::char_width::char_width_at;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code};
 use uucore::{format_usage, show, translate};
@@ -361,17 +361,8 @@ fn classify_char(buf: &[u8], byte: usize, utf8: bool) -> (CharType, usize, usize
     }
 
     if utf8 {
-        let nbytes = char::from(b).len_utf8();
-        let Some(slice) = buf.get(byte..byte + nbytes) else {
-            // don't overrun buffer because of invalid UTF-8
-            return (Other, 1, 1);
-        };
-
-        if let Ok(t) = from_utf8(slice)
-            && let Some(c) = t.chars().next()
-        {
-            return (Other, UnicodeWidthChar::width(c).unwrap_or(0), nbytes);
-        }
+        let (width, nbytes) = char_width_at(buf, byte);
+        return (Other, width, nbytes);
     }
     (Other, 1, 1) // implicit assumption: non-UTF-8 char is 1 col wide
 }

src/uu/unexpand/src/unexpand.rs

@@ -3,16 +3,16 @@
 // For the full copyright and license information, please view the LICENSE
 // file that was distributed with this source code.
 
-// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess
+// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess iswblank
 
 use clap::{Arg, ArgAction, Command};
 use std::ffi::OsString;
 use std::fs::File;
 use std::io::{self, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout};
 use std::num::IntErrorKind;
 use std::path::Path;
-use std::str::from_utf8;
 use thiserror::Error;
+use uucore::char_width::char_info_at;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code};
 use uucore::translate;
@@ -374,19 +374,47 @@ fn write_tabs(
             && (print_state.leading || ai && print_state.pctype == CharType::Tab))
     {
         while let Some(nts) = next_tabstop(tab_config, print_state.scol) {
-            if print_state.col < print_state.scol + nts {
+            let target = print_state.scol + nts;
+            if print_state.col < target {
+                break;
+            }
+            // A tab may absorb a wide blank that lies fully within it, but not
+            // one that straddles the tab stop; stop converting before it.
+            if print_state
+                .pending_wide
+                .iter()
+                .any(|&(start, width, _)| start < target && target < start + width)
+            {
                 break;
             }
 
             output.write_all(b"\t")?;
-            print_state.scol += nts;
+            print_state.scol = target;
         }
     }
 
+    // Fill the remaining columns. Wide blanks that were not absorbed into a tab
+    // are emitted verbatim; everything else is a plain space.
+    let mut wide_idx = 0;
     while print_state.col > print_state.scol {
-        output.write_all(b" ")?;
-        print_state.scol += 1;
+        while wide_idx < print_state.pending_wide.len()
+            && print_state.pending_wide[wide_idx].0 < print_state.scol
+        {
+            wide_idx += 1;
+        }
+        if wide_idx < print_state.pending_wide.len()
+            && print_state.pending_wide[wide_idx].0 == print_state.scol
+        {
+            let (_, width, bytes) = &print_state.pending_wide[wide_idx];
+            output.write_all(bytes)?;
+            print_state.scol += width;
+            wide_idx += 1;
+        } else {
+            output.write_all(b" ")?;
+            print_state.scol += 1;
+        }
     }
+    print_state.pending_wide.clear();
     Ok(())
 }
 
@@ -398,27 +426,42 @@ enum CharType {
     Other,
 }
 
-fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usize) {
+/// A multibyte blank is a horizontal space other than ASCII space/tab, i.e. a
+/// character for which `iswblank` is true in a UTF-8 locale. Such a blank
+/// participates in tab conversion (its columns can be absorbed into a tab), so
+/// it must be classified as [`CharType::Space`]. The set is the Unicode space
+/// separators minus the non-breaking ones (U+00A0, U+2007, U+202F).
+fn is_multibyte_blank(c: char) -> bool {
+    matches!(c,
+        '\u{1680}'
+        | '\u{2000}'..='\u{2006}'
+        | '\u{2008}'..='\u{200A}'
+        | '\u{205F}'
+        | '\u{3000}')
+}
+
+/// Classify the character at `buf[byte]` for tab conversion, returning its
+/// [`CharType`] along with its display width and byte length. Unlike
+/// [`char_info_at`], the first element is a classification rather than the
+/// decoded `char`.
+fn classify_char_at(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usize) {
     use CharType::{Backspace, Other, Space, Tab};
     let b = buf[byte];
     if b.is_ascii() {
         return match b {
-            b' ' => (Space, 0, 1),
+            b' ' => (Space, 1, 1),
             b'\t' => (Tab, 0, 1),
             b'\x08' => (Backspace, 0, 1),
             _ => (Other, 1, 1),
         };
     }
 
     if utf8 {
-        let nbytes = char::from(b).len_utf8();
-        // don't overrun the buffer because of invalid UTF-8
-        if buf
-            .get(byte..byte + nbytes)
-            .is_some_and(|s| from_utf8(s).is_ok())
-        {
-            return (Other, nbytes, nbytes);
+        let (c, width, nbytes) = char_info_at(buf, byte);
+        if c.is_some_and(is_multibyte_blank) {
+            return (Space, width, nbytes);
         }
+        return (Other, width, nbytes);
     }
     (Other, 1, 1)
 }
@@ -434,6 +477,10 @@ struct PrintState {
     scol: usize,
     leading: bool,
     pctype: CharType,
+    // Multibyte blanks buffered but not yet written, as (start column, display
+    // width, bytes). They are absorbed into a tab when a run reaches a tab stop
+    // and otherwise emitted verbatim by write_tabs. Empty for ASCII-only input.
+    pending_wide: Vec<(usize, usize, Vec<u8>)>,
 }
 
 impl PrintState {
@@ -443,6 +490,7 @@ impl PrintState {
         self.scol = 0;
         self.leading = true;
         self.pctype = CharType::Other;
+        self.pending_wide.clear();
     }
 }
 
@@ -475,36 +523,49 @@ fn unexpand_buf(
 
     // We can only fast forward if we don't need to calculate col/scol
     if let Some(b'\n') = buf.last() {
-        // Fast path for leading spaces in non-UTF8 mode: count consecutive spaces/tabs at start
-        if !options.utf8 && !options.aflag && print_state.leading {
-            // In default mode (not -a), we only convert leading spaces
-            // So we can batch process them and then copy the rest
-            while byte < buf.len() {
-                match buf[byte] {
-                    b' ' => {
-                        print_state.col += 1;
-                        byte += 1;
-                    }
+        // Fast path: count consecutive leading spaces/tabs at the start of the
+        // line. Leading whitespace is ASCII, so this is independent of UTF-8
+        // mode; the rest of the line is copied verbatim in default mode. It does
+        // not apply once a finite last column is in play (multiple tab stops),
+        // where the general loop stops converting past that column.
+        if !options.aflag && print_state.leading && lastcol == 0 {
+            // In default mode (not -a), we only convert leading spaces, so batch
+            // over them and then copy the rest of the line. Compute the column
+            // into a local without touching state: if the first non-blank is a
+            // multibyte start it may be a blank that needs the general loop, so
+            // bail out to it instead of committing here.
+            let mut end = 0;
+            let mut col = print_state.col;
+            let mut saw_tab = false;
+            while end < buf.len() {
+                match buf[end] {
+                    b' ' => col += 1,
                     b'\t' => {
-                        print_state.col += next_tabstop(tab_config, print_state.col).unwrap_or(1);
-                        byte += 1;
-                        print_state.pctype = CharType::Tab;
+                        col += next_tabstop(tab_config, col).unwrap_or(1);
+                        saw_tab = true;
                     }
                     _ => break,
                 }
+                end += 1;
             }
+            if end == buf.len() || buf[end].is_ascii() {
+                print_state.col = col;
+                if saw_tab {
+                    print_state.pctype = CharType::Tab;
+                }
 
-            // If we found spaces/tabs, write them as tabs
-            if byte > 0 {
-                write_tabs(output, tab_config, print_state, options.aflag)?;
-            }
+                // If we found spaces/tabs, write them as tabs
+                if end > 0 {
+                    write_tabs(output, tab_config, print_state, options.aflag)?;
+                }
 
-            // Write the rest of the line directly (no more tab conversion needed)
-            if byte < buf.len() {
-                print_state.leading = false;
-                output.write_all(&buf[byte..])?;
+                // Write the rest of the line directly (no more tab conversion needed)
+                if end < buf.len() {
+                    print_state.leading = false;
+                    output.write_all(&buf[end..])?;
+                }
+                return Ok(());
             }
-            return Ok(());
         }
     }
 
@@ -518,15 +579,25 @@ fn unexpand_buf(
         }
 
         // figure out how big the next char is, if it's UTF-8
-        let (ctype, cwidth, nbytes) = next_char_info(options.utf8, buf, byte);
+        let (ctype, cwidth, nbytes) = classify_char_at(options.utf8, buf, byte);
 
         // now figure out how many columns this char takes up, and maybe print it
         let tabs_buffered = print_state.leading || options.aflag;
         match ctype {
             CharType::Space | CharType::Tab => {
+                // A buffered multibyte blank must be remembered so it can be
+                // emitted verbatim if its run is not converted to a tab.
+                if tabs_buffered && ctype == CharType::Space && nbytes > 1 {
+                    print_state.pending_wide.push((
+                        print_state.col,
+                        cwidth,
+                        buf[byte..byte + nbytes].to_vec(),
+                    ));
+                }
+
                 // compute next col, but only write space or tab chars if not buffering
                 print_state.col += if ctype == CharType::Space {
-                    1
+                    cwidth
                 } else {
                     next_tabstop(tab_config, print_state.col).unwrap_or(1)
                 };
@@ -575,6 +646,7 @@ fn unexpand_file(
         scol: 0,
         leading: true,
         pctype: CharType::Other,
+        pending_wide: Vec::new(),
     };
 
     loop {

src/uucore/Cargo.toml

@@ -49,6 +49,10 @@ z85 = { version = "3.0.5", optional = true }
 base64-simd = { version = "0.8", optional = true }
 libc = { workspace = true, optional = true }
 os_display = "0.1.3"
+# Not optional: os_display already pulls unicode-width into every uucore build,
+# so making it a direct dependency here is free and keeps char_width available
+# on all targets (a feature-gated optional dep failed to activate on wasm).
+unicode-width = { workspace = true }
 
 # Benchmark dependencies (optional)
 divan = { workspace = true, optional = true }

src/uucore/src/lib/features.rs

@@ -12,6 +12,7 @@ pub mod backup_control;
 pub mod benchmark;
 #[cfg(feature = "buf-copy")]
 pub mod buf_copy;
+pub mod char_width;
 #[cfg(feature = "checksum")]
 pub mod checksum;
 #[cfg(feature = "colors")]