columnar_menu panics on non-ASCII characters in directory

[sarubo]

Platform Windows 10, Arch Linux
Terminal software Windows Terminal, Alacritty

nu > version | transpose key value | to md --pretty

key	value
version	0.91.1
branch	main
commit_hash	6e37ad0275cb23c5a8a933ba6a46a45d70f74910
build_os	windows-x86_64
build_target	x86_64-pc-windows-gnu
rust_version	rustc 1.74.1 (a28077b28 2023-12-04)
rust_channel	1.74.1-x86_64-pc-windows-gnu
cargo_version	cargo 1.74.1 (ecb9851af 2023-10-18)
build_time	2024-03-19 18:57:34 +09:00
build_rust_channel	debug
allocator	mimalloc
features	default, sqlite, trash, which
installed_plugins

With RUST_BACKTRACE=1 it will look like this:

sarub@LAPTOP-WIN-SRB:~/projects/rust/nushell/target/debug
nu > cd `foo/`Error:   × Main thread panicked.
  ├─▶ at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\str\mod.rs:660:13
  ╰─▶ byte index 6 is not a char boundary; it is inside 'あ' (bytes 4..7) of `foo\あ\`
         0: 0x7ff62799c550 - rust_begin_unwind
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library\std\src/panicking.rs:597
         1: 0x7ff6279f77f7 - core::panicking::panic_fmt::hd44cd0ce6cc3cd33
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library\core\src/panicking.rs:72
         2: 0x7ff627a006e7 - core::str::slice_error_fail_rt::h9eebe5afeb115af2
         3: 0x7ff627a001c3 - core::str::slice_error_fail::h99c8592d623ec1b6
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library\core\src\str/mod.rs:87
         4: 0x7ff625db595b - core::str::<impl str>::split_at::he586fec1974bbb3e
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\str/mod.rs:660
         5: 0x7ff625e2db33 - reedline::menu::columnar_menu::ColumnarMenu::create_string::h3c11804425383c10
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\menu\columnar_menu.rs:305
         6: 0x7ff625e34cd4 - <reedline::menu::columnar_menu::ColumnarMenu as reedline::menu::Menu>::menu_string::{{closure}}::hd085b8638d8784e4
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\menu\columnar_menu.rs:659
         7: 0x7ff625e438fe - core::iter::adapters::map::map_fold::{{closure}}::h9e115fbce619a1c8
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/map.rs:84
         8: 0x7ff625e4ad17 - <core::iter::adapters::enumerate::Enumerate<I> as core::iter::traits::iterator::Iterator>::fold::enumerate::{{closure}}::h28ff5ad5f97eb1fd
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/enumerate.rs:107
         9: 0x7ff625d6f22d - core::ops::try_trait::NeverShortCircuit<T>::wrap_mut_2::{{closure}}::h5c51c4c418648308
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\ops/try_trait.rs:417
        10: 0x7ff625e3c436 - <core::iter::adapters::take::Take<I> as core::iter::traits::iterator::Iterator>::try_fold::check::{{closure}}::hbded6ce406225366
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/take.rs:90
        11: 0x7ff625df7e4c - core::iter::traits::iterator::Iterator::try_fold::h386febb3f820b158
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\traits/iterator.rs:2461
        12: 0x7ff625db2f79 - <core::iter::adapters::skip::Skip<I> as core::iter::traits::iterator::Iterator>::try_fold::h23405cb4fcf7075a
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/skip.rs:113
        13: 0x7ff625e3b706 - <core::iter::adapters::take::Take<I> as core::iter::traits::iterator::Iterator>::try_fold::h97f523b0524d6090
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/take.rs:99
        14: 0x7ff625e45a38 - <core::iter::adapters::take::Take<I> as core::iter::adapters::take::SpecTake>::spec_fold::h2a2a6868167d3759
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/take.rs:264
        15: 0x7ff625e3ae69 - <core::iter::adapters::take::Take<I> as core::iter::traits::iterator::Iterator>::fold::h31b9ae40e2065444
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/take.rs:109
        16: 0x7ff625e4aa16 - <core::iter::adapters::enumerate::Enumerate<I> as core::iter::traits::iterator::Iterator>::fold::h042c5eedb7591a4c
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/enumerate.rs:113
        17: 0x7ff625e3ddf9 - <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold::hd3e89ccd89b84fb5
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\adapters/map.rs:124
        18: 0x7ff625e420d3 - core::iter::traits::iterator::Iterator::for_each::h6f7265366285484b
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\traits/iterator.rs:857
        19: 0x7ff625d6dd36 - <alloc::string::String as core::iter::traits::collect::Extend<alloc::string::String>>::extend::h563773edc5b2190f
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\alloc\src/string.rs:2081
                       - <alloc::string::String as core::iter::traits::collect::FromIterator<alloc::string::String>>::from_iter::h2650f0f4b0c287a3
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\alloc\src/string.rs:1980
        20: 0x7ff625e41a11 - core::iter::traits::iterator::Iterator::collect::h43170591e86ac5a3
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\iter\traits/iterator.rs:2053
        21: 0x7ff625e34b49 - <reedline::menu::columnar_menu::ColumnarMenu as reedline::menu::Menu>::menu_string::h28881a3823acca99
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\menu\columnar_menu.rs:648
        22: 0x7ff625e5b186 - <reedline::menu::ReedlineMenu as reedline::menu::Menu>::menu_string::ha23bc485978682d2
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\menu\mod.rs:457
        23: 0x7ff625df005e - reedline::painting::painter::Painter::print_menu::hcbe6e5cd0f4aa264
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\painting\painter.rs:249
        24: 0x7ff625df0b95 - reedline::painting::painter::Painter::print_small_buffer::h6096b69e349c457a
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\painting\painter.rs:301
        25: 0x7ff625def3dc - reedline::painting::painter::Painter::repaint_buffer::h13ed65d0abc6f7e3
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\painting\painter.rs:181
        26: 0x7ff625e11587 - reedline::engine::Reedline::buffer_paint::hd921209da57cace8
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\engine.rs:1735
        27: 0x7ff625e0ec6d - reedline::engine::Reedline::repaint::h0db83fd267c3c251
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\engine.rs:1489
        28: 0x7ff625e08431 - reedline::engine::Reedline::read_line_helper::h4646a8964a18c1ce
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\engine.rs:784
        29: 0x7ff625e076e4 - reedline::engine::Reedline::read_line::h782a52f81cbb3235
                      at C:\Users\sarub\.cargo\git\checkouts\reedline-e42026a78d91c510\dc7063e\src\engine.rs:633
        30: 0x7ff625c03edb - nu_cli::repl::loop_iteration::h4d35fddeae4e2880
                      at C:\Users\sarub\projects\rust\nushell\crates\nu-cli\src\repl.rs:498
        31: 0x7ff625bb807c - nu_cli::repl::evaluate_repl::{{closure}}::h873a03627ad5c39d
                      at C:\Users\sarub\projects\rust\nushell\crates\nu-cli\src\repl.rs:147
        32: 0x7ff625c8f9f0 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h62b164b3bda91959
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\core\src\panic/unwind_safe.rs:271
        33: 0x7ff625c8acb7 - std::panicking::try::do_call::h5d7abccbcb0bb082
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\std\src/panicking.rs:504
        34: 0x7ff625c8ada8 - __rust_try
        35: 0x7ff625c8abd7 - std::panicking::try::hea61009784c5a1da
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\std\src/panicking.rs:468
        36: 0x7ff625c0d1f1 - std::panic::catch_unwind::h16a9083e820817e1
                      at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1\library\std\src/panic.rs:142
        37: 0x7ff625c00bb1 - nu_cli::repl::evaluate_repl::h6f0c7d4e4815a16d
                      at C:\Users\sarub\projects\rust\nushell\crates\nu-cli\src\repl.rs:146
        38: 0x7ff623daca16 - nu::run::run_repl::he36ca1a18c5d5d2b
                      at C:\Users\sarub\projects\rust\nushell\src\run.rs:273
        39: 0x7ff623d88683 - nu::main::hbcfc0b6f3c945ebc
                      at C:\Users\sarub\projects\rust\nushell\src\main.rs:384
        40: 0x7ff623dad9bb - core::ops::function::FnOnce::call_once::h4180148fe422dd29
## main...origin/main
03/19(火)午後07:01.44
sarub@LAPTOP-WIN-SRB:~/projects/rust/nushell/target/debug
| cd `foo/`
## main...origin/main
03/19(火)午後07:02.03
sarub@LAPTOP-WIN-SRB:~/projects/rust/nushell/target/debug
nu >

Related to #769. Especially #769 (comment)

i think it might be this line here:

reedline/src/menu/columnar_menu.rs

Line 305 in 0698712

let (match_str, remaining_str) = suggestion.value.split_at(match_len);

we could probably add a check there and if it fails we just dont style (looks like that can fail with some external completers)

columnar_menu panics with or without external completers.

Steps to reproduce

1. Create multiple directories or files containing non-ASCII characters.
   1. mkdir `foo/bar/`
   2. mkdir `foo/あ/`
2. If you type cd `foo/` in nushell and press the tab key, it will panic.

• columnar_menu does not panic if it is cd foo/ instead of cd `foo/` .
• columnar_menu does not create foo/bar/ and does not panic if it just creates foo/あ/.
• columnar_menu does not panic if directory is foo/baz/ instead of foo/あ/.
• columnar_menu does not panic after moving to foo.

Screenshots/Screencaptures