imgcat: work better with tmux and conpty

Neither of these understand image protocols, and both are
an additional processing layer between the application and
wezterm.

This commit detects and wraps OSC sequences in tmux's passthru
sequence so that the data is passed on to wezterm rather than
elided from the data stream.

For image protocols in both tmux and conpty, work a little
smarter and explicitly move the cursor position to the same
location that wezterm would move it to.  That prevents the
display from being as mangled by tmux/conpty due to a diverging
understanding of the cursor position.

The logic isn't perfect, and can result in the x-coordinate
being incorrect, and this won't work with the new --position
argument either in its current state, without adding a lot
of complexity to deal with scrolling and relative and absolute
positioning handling.

To facilitate that, a new termwiz Terminal trait method has
been added to probe the terminal name, version, cell and pixel
dimensions. It's not pretty.

refs: https://github.com/wez/wezterm/issues/3624
refs: https://github.com/wez/wezterm/issues/3716

Cargo.lock

@@ -2369,8 +2369,9 @@ dependencies = [
 
 [[package]]
 name = "image"
-version = "0.24.5"
-source = "git+https://github.com/image-rs/image.git?rev=fe069785ae245a2c510fd724ef96da283b05a236#fe069785ae245a2c510fd724ef96da283b05a236"
+version = "0.24.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "527909aa81e20ac3a44803521443a765550f09b5130c2c2fa1ea59c2f8f50a3a"
 dependencies = [
  "bytemuck",
  "byteorder",
@@ -5840,6 +5841,7 @@ dependencies = [
  "env-bootstrap",
  "filedescriptor",
  "hostname",
+ "image",
  "libc",
  "log",
  "mux",

docs/changelog.md

@@ -27,6 +27,20 @@ As features stabilize some brief notes about them will accumulate here.
 #### New
 * [wezterm imgcat](cli/imgcat.md) now has `--position`, `--no-move-cursor` and
   `--hold` options. #3716
+* [wezterm set-working-directory](cli/set-working-directory.md) will now wrap
+  its OSC 7 escape sequence in the tmux passthru sequence when necessary. This can be
+  controlled via new `--tmux-passthru` option.
+* [wezterm imgcat](cli/imgcat.md) will now wrap the image transfer OSC escape
+  sequences in the tmux passthru sequence when necessary. This can be
+  controlled via new `--tmux-passthru` option. Note that tmux doesn't natively
+  understand these sequences, and tmux will wipe out the image when it redraws
+  the screen as part of scrolling back through its history. imgcat support in
+  tmux is very basic effort
+* [wezterm imgcat](cli/imgcat.md) will compensate for tmux and conpty, which
+  do not natively understand image protocols, and adjust the cursor position
+  in order to avoid the shell/prompt from mangling the image after it is printing.
+  Support for this has limitations and will not take effect when the new
+  `--position` argument is used. #3624
 
 #### Fixed
 * Command Palette was using now-invalid Nerd Font 2.0 symbols for macOS

lua-api-crates/color-funcs/Cargo.toml

@@ -10,8 +10,7 @@ anyhow = "1.0"
 config = { path = "../../config" }
 csscolorparser = {version="0.6", features=["lab"]}
 deltae = "0.3"
-# we want image 0.24.6 or later. see https://github.com/wez/wezterm/issues/3250
-image = {version="0.24", git="https://github.com/image-rs/image.git", rev="fe069785ae245a2c510fd724ef96da283b05a236"}
+image = "0.24.6"
 lazy_static = "1.4"
 log = "0.4"
 lru = "0.7"

mux/src/ssh.rs

@@ -511,6 +511,10 @@ fn connect_ssh_session(
             })
         }
 
+        fn probe_screen_size(&mut self) -> termwiz::Result<ScreenSize> {
+            self.get_screen_size()
+        }
+
         fn set_screen_size(&mut self, _size: ScreenSize) -> termwiz::Result<()> {
             termwiz::bail!("TerminalShim cannot set screen size");
         }

mux/src/termwiztermtab.rs

@@ -397,6 +397,10 @@ impl termwiz::terminal::Terminal for TermWizTerminal {
         Ok(self.render_tx.screen_size)
     }
 
+    fn probe_screen_size(&mut self) -> termwiz::Result<ScreenSize> {
+        Ok(self.render_tx.screen_size)
+    }
+
     fn set_screen_size(&mut self, _size: ScreenSize) -> termwiz::Result<()> {
         termwiz::bail!("TermWizTerminalPane cannot set screen size");
     }

term/Cargo.toml

@@ -22,8 +22,7 @@ humansize = "2.1"
 miniz_oxide = "0.4"
 finl_unicode = "1.2"
 hex = "0.4"
-# we want image 0.24.6 or later. see https://github.com/wez/wezterm/issues/3250
-image = {version="0.24", git="https://github.com/image-rs/image.git", rev="fe069785ae245a2c510fd724ef96da283b05a236"}
+image = "0.24.6"
 lazy_static = "1.4"
 log = "0.4"
 lru = "0.7"

termwiz/Cargo.toml

@@ -22,8 +22,7 @@ finl_unicode = "1.2"
 fixedbitset = "0.4"
 fnv = {version="1.0", optional=true}
 hex = "0.4"
-# we want image 0.24.6 or later. see https://github.com/wez/wezterm/issues/3250
-image = {version="0.24", git="https://github.com/image-rs/image.git", rev="fe069785ae245a2c510fd724ef96da283b05a236", optional=true}
+image = {version="0.24.6", optional=true}
 lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"

termwiz/src/render/terminfo.rs

@@ -870,6 +870,10 @@ mod test {
             })
         }
 
+        fn probe_screen_size(&mut self) -> Result<ScreenSize> {
+            self.get_screen_size()
+        }
+
         fn set_screen_size(&mut self, size: ScreenSize) -> Result<()> {
             let size = winsize {
                 ws_row: cast(size.rows)?,

termwiz/src/terminal/mod.rs

@@ -6,6 +6,7 @@ use crate::surface::Change;
 use crate::{format_err, Result};
 use num_traits::NumCast;
 use std::fmt::Display;
+use std::io::{Read, Write};
 use std::time::Duration;
 
 #[cfg(unix)]
@@ -40,6 +41,154 @@ pub struct ScreenSize {
     pub ypixel: usize,
 }
 
+impl ScreenSize {
+    /// This is a helper function to facilitate the implementation of
+    /// Terminal::probe_screen_size. It will emit a series of terminal
+    /// escape sequences intended to probe the terminal and determine
+    /// the ScreenSize.
+    /// `input` and `output` are the corresponding reading and writable
+    /// handles to the terminal.
+    pub fn probe<I: Read, O: Write>(mut input: I, mut output: O) -> Result<Self> {
+        use crate::escape::csi::{Device, Window};
+        use crate::escape::parser::Parser;
+        use crate::escape::{Action, DeviceControlMode, Esc, EscCode, CSI};
+
+        let xt_version = CSI::Device(Box::new(Device::RequestTerminalNameAndVersion));
+        let query_cells = CSI::Window(Box::new(Window::ReportTextAreaSizeCells));
+        let query_pixels = CSI::Window(Box::new(Window::ReportCellSizePixels));
+        let dev_attributes = CSI::Device(Box::new(Device::RequestPrimaryDeviceAttributes));
+
+        // some tmux versions have their rows/cols swapped in ReportTextAreaSizeCells,
+        // so we need to figure out the specific tmux version we're talking to
+        write!(output, "{xt_version}{dev_attributes}")?;
+        output.flush()?;
+        let mut term = vec![];
+        let mut parser = Parser::new();
+        let mut done = false;
+
+        while !done {
+            let mut byte = [0u8];
+            input.read(&mut byte)?;
+
+            parser.parse(&byte, |action| {
+                // print!("{action:?}\r\n");
+                match action {
+                    Action::Esc(Esc::Code(EscCode::StringTerminator)) => {}
+                    Action::DeviceControl(dev) => {
+                        if let DeviceControlMode::Data(b) = dev {
+                            term.push(b);
+                        }
+                    }
+                    _ => {
+                        done = true;
+                    }
+                }
+            });
+        }
+
+        /*
+        print!(
+            "probed terminal version: {}\r\n",
+            String::from_utf8_lossy(&term)
+        );
+        */
+
+        let is_tmux = term.starts_with(b"tmux ");
+        let swapped_cols_rows = if is_tmux {
+            let version = &term[5..];
+            match version {
+                b"3.2" | b"3.2a" | b"3.3" | b"3.3a" => true,
+                _ => false,
+            }
+        } else {
+            false
+        };
+
+        write!(output, "{query_cells}{query_pixels}")?;
+
+        // tmux refuses to directly support responding to 14t or 16t queries
+        // for pixel dimensions, so we need to jump through to the outer
+        // terminal and see what it says
+        if is_tmux {
+            let tmux_begin = "\u{1b}Ptmux;\u{1b}";
+            let tmux_end = "\u{1b}\\";
+            write!(output, "{tmux_begin}{query_pixels}{tmux_end}")?;
+            output.flush()?;
+            // I really wanted to avoid a delay here, but tmux will re-order the
+            // response to dev_attributes before it sends the response for the
+            // passthru of query_pixels if we don't delay. The delay is potentially
+            // imperfect for things like a laggy ssh connection. The consequence
+            // of the timing being wrong is that we won't be able to reason about
+            // the pixel dimensions, which is "OK", but that was kinda the whole
+            // point of probing this way vs. termios.
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+
+        write!(output, "{dev_attributes}")?;
+        output.flush()?;
+
+        let mut parser = Parser::new();
+        let mut done = false;
+        let mut size = ScreenSize {
+            rows: 0,
+            cols: 0,
+            xpixel: 0,
+            ypixel: 0,
+        };
+
+        while !done {
+            let mut byte = [0u8];
+            input.read(&mut byte)?;
+
+            parser.parse(&byte, |action| {
+                // print!("{action:?}\r\n");
+                match action {
+                    Action::CSI(csi) => match csi {
+                        CSI::Window(win) => match *win {
+                            Window::ResizeWindowCells { width, height } => {
+                                let width = width.unwrap_or(1);
+                                let height = height.unwrap_or(1);
+                                if width > 0 && height > 0 {
+                                    let width = width as usize;
+                                    let height = height as usize;
+                                    if swapped_cols_rows {
+                                        size.rows = width;
+                                        size.cols = height;
+                                    } else {
+                                        size.rows = height;
+                                        size.cols = width;
+                                    }
+                                }
+                            }
+                            Window::ReportCellSizePixelsResponse { width, height } => {
+                                let width = width.unwrap_or(1);
+                                let height = height.unwrap_or(1);
+                                if width > 0 && height > 0 {
+                                    let width = width as usize;
+                                    let height = height as usize;
+                                    size.xpixel = width;
+                                    size.ypixel = height;
+                                }
+                            }
+                            _ => {
+                                done = true;
+                            }
+                        },
+                        _ => {
+                            done = true;
+                        }
+                    },
+                    _ => {
+                        done = true;
+                    }
+                }
+            });
+        }
+
+        Ok(size)
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Blocking {
     DoNotWait,
@@ -68,6 +217,12 @@ pub trait Terminal {
     /// Queries the current screen size, returning width, height.
     fn get_screen_size(&mut self) -> Result<ScreenSize>;
 
+    /// Like get_screen_size but uses escape sequences to interrogate
+    /// the terminal rather than relying on the termios/kernel interface
+    /// You should delegate this to `ScreenSize::probe(&mut self.read, &mut self.write)`
+    /// to implement this method.
+    fn probe_screen_size(&mut self) -> Result<ScreenSize>;
+
     /// Sets the current screen size
     fn set_screen_size(&mut self, size: ScreenSize) -> Result<()>;
 

termwiz/src/terminal/unix.rs

@@ -385,6 +385,10 @@ impl Terminal for UnixTerminal {
         })
     }
 
+    fn probe_screen_size(&mut self) -> Result<ScreenSize> {
+        ScreenSize::probe(&mut self.read, &mut self.write)
+    }
+
     fn set_screen_size(&mut self, size: ScreenSize) -> Result<()> {
         let size = winsize {
             ws_row: cast(size.rows)?,

termwiz/src/terminal/windows.rs

@@ -769,6 +769,10 @@ impl Terminal for WindowsTerminal {
         })
     }
 
+    fn probe_screen_size(&mut self) -> Result<ScreenSize> {
+        ScreenSize::probe(&mut self.input_handle, &mut self.output_handle)
+    }
+
     fn set_screen_size(&mut self, size: ScreenSize) -> Result<()> {
         // FIXME: take into account the visible window size here;
         // this probably changes the size of everything including scrollback

wezterm-gui/Cargo.toml

@@ -55,8 +55,7 @@ futures = "0.3"
 fuzzy-matcher = "0.3"
 hdrhistogram = "7.1"
 http_req = "0.9"
-# we want image 0.24.6 or later. see https://github.com/wez/wezterm/issues/3250
-image = {version="0.24", git="https://github.com/image-rs/image.git", rev="fe069785ae245a2c510fd724ef96da283b05a236"}
+image = "0.24.6"
 lazy_static = "1.4"
 libc = "0.2"
 lfucache = { path = "../lfucache" }

wezterm/Cargo.toml

@@ -17,6 +17,7 @@ config = { path = "../config" }
 env-bootstrap = { path = "../env-bootstrap" }
 filedescriptor = { version="0.8", path = "../filedescriptor" }
 hostname = "0.3"
+image = "0.24.6"
 libc = "0.2"
 log = "0.4"
 mux = { path = "../mux" }

wezterm/src/main.rs

@@ -10,7 +10,9 @@ use termwiz::caps::Capabilities;
 use termwiz::escape::esc::{Esc, EscCode};
 use termwiz::escape::OneBased;
 use termwiz::input::{InputEvent, KeyCode, KeyEvent};
-use termwiz::terminal::Terminal;
+use termwiz::surface::change::Change;
+use termwiz::surface::Position;
+use termwiz::terminal::{ScreenSize, Terminal};
 use umask::UmaskSaver;
 use wezterm_gui_subcommands::*;
 
@@ -177,6 +179,10 @@ struct ImgCatCommand {
     #[arg(long)]
     hold: bool,
 
+    /// How to manage passing the escape through to tmux
+    #[arg(long, value_parser)]
+    tmux_passthru: Option<TmuxPassthru>,
+
     /// The name of the image file to be displayed.
     /// If omitted, will attempt to read it from stdin.
     #[arg(value_parser, value_hint=ValueHint::FilePath)]
@@ -204,7 +210,97 @@ fn x_comma_y(arg: &str) -> Result<ImagePosition, String> {
     }
 }
 
+#[derive(Debug)]
+pub(crate) struct ImageInfo {
+    pub width: u32,
+    pub height: u32,
+    pub _format: image::ImageFormat,
+}
+
 impl ImgCatCommand {
+    fn compute_image_cell_dimensions(
+        &self,
+        info: ImageInfo,
+        term_size: ScreenSize,
+    ) -> (usize, usize) {
+        let physical_cols = term_size.cols;
+        let physical_rows = term_size.rows;
+        let cell_pixel_width = term_size.xpixel;
+        let cell_pixel_height = term_size.ypixel;
+        let pixel_width = cell_pixel_width * physical_cols;
+        let pixel_height = cell_pixel_height * physical_rows;
+
+        let width = self
+            .width
+            .unwrap_or_default()
+            .to_pixels(cell_pixel_width, physical_cols);
+        let height = self
+            .height
+            .unwrap_or_default()
+            .to_pixels(cell_pixel_height, physical_rows);
+
+        // Compute any Automatic dimensions
+        let aspect = info.width as f32 / info.height as f32;
+
+        let (width, height) = match (width, height) {
+            (None, None) => {
+                // Take the image's native size
+                let width = info.width as usize;
+                let height = info.height as usize;
+                // but ensure that it fits
+                if width as usize > pixel_width || height as usize > pixel_height {
+                    let width = width as f32;
+                    let height = height as f32;
+                    let mut candidates = vec![];
+
+                    let x_scale = pixel_width as f32 / width;
+                    if height * x_scale <= pixel_height as f32 {
+                        candidates.push((pixel_width, (height * x_scale) as usize));
+                    }
+                    let y_scale = pixel_height as f32 / height;
+                    if width * y_scale <= pixel_width as f32 {
+                        candidates.push(((width * y_scale) as usize, pixel_height));
+                    }
+
+                    candidates.sort_by(|a, b| (a.0 * a.1).cmp(&(b.0 * b.1)));
+
+                    candidates.pop().unwrap()
+                } else {
+                    (width, height)
+                }
+            }
+            (Some(w), None) => {
+                let h = w as f32 / aspect;
+                (w, h as usize)
+            }
+            (None, Some(h)) => {
+                let w = h as f32 * aspect;
+                (w as usize, h)
+            }
+            (Some(w), Some(_)) if !self.no_preserve_aspect_ratio => {
+                let h = w as f32 / aspect;
+                (w, h as usize)
+            }
+            (Some(w), Some(h)) => (w, h),
+        };
+
+        // And convert to cells
+        (width / cell_pixel_width, height / cell_pixel_height)
+    }
+
+    fn image_dimensions(data: &[u8]) -> anyhow::Result<ImageInfo> {
+        let reader = image::io::Reader::new(std::io::Cursor::new(data)).with_guessed_format()?;
+        let format = reader
+            .format()
+            .ok_or_else(|| anyhow::anyhow!("unknown format!?"))?;
+        let (width, height) = reader.into_dimensions()?;
+        Ok(ImageInfo {
+            width,
+            height,
+            _format: format,
+        })
+    }
+
     fn run(&self) -> anyhow::Result<()> {
         let mut data = Vec::new();
         if let Some(file_name) = self.file_name.as_ref() {
@@ -216,9 +312,35 @@ impl ImgCatCommand {
             stdin.read_to_end(&mut data)?;
         }
 
-        if let Some(position) = &self.position {
-            let save_cursor = Esc::Code(EscCode::DecSaveCursorPosition);
+        // TODO: ideally we'd provide some kind of ProbeCapabilities type
+        // that can be returned from Terminal that will encode this sort
+        // of thing so that we can use xtversion to know for sure.
+        let is_tmux = TmuxPassthru::is_tmux();
 
+        // TODO: ideally we'd do some kind of probing to see if conpty
+        // is in the mix. For now we just assume that if we are on windows
+        // then it must be in there somewhere.
+        let is_conpty = cfg!(windows);
+
+        // Not all systems understand that the cursor should move as
+        // part of processing the image escapes, so we need to move it
+        // explicitly after we've drawn things.
+        // We can only do this reasonably sanely if we aren't setting
+        // the absolute position.
+        let needs_force_cursor_move =
+            !self.no_move_cursor && !self.position.is_some() && (is_tmux || is_conpty);
+
+        let caps = Capabilities::new_from_env()?;
+        let mut term = termwiz::terminal::new_terminal(caps)?;
+        term.set_raw_mode()?;
+
+        let term_size = term.probe_screen_size()?;
+        term.set_cooked_mode()?;
+
+        let save_cursor = Esc::Code(EscCode::DecSaveCursorPosition);
+        let restore_cursor = Esc::Code(EscCode::DecRestoreCursorPosition);
+
+        if let Some(position) = &self.position {
             let csi = termwiz::escape::CSI::Cursor(
                 termwiz::escape::csi::Cursor::CharacterAndLinePosition {
                     col: OneBased::from_zero_based(position.x),
@@ -228,6 +350,26 @@ impl ImgCatCommand {
             print!("{save_cursor}{csi}");
         }
 
+        let (begin, end) = self.tmux_passthru.unwrap_or_default().get();
+
+        let image_dims = Self::image_dimensions(&data)
+            .map(|info| self.compute_image_cell_dimensions(info, term_size));
+
+        if let (Ok((_cursor_x, cursor_y)), true) = (&image_dims, needs_force_cursor_move) {
+            // Before we emit the image, we need to emit some new lines so that
+            // if the image would scroll the display, things end up in the right place
+            let new_lines = "\n".repeat(*cursor_y);
+            print!("{new_lines}");
+
+            // and move back up again.
+            // Note that we lose the x cursor position and land back in the first
+            // column as a result of doing this.
+            term.render(&[Change::CursorPosition {
+                x: Position::Absolute(0),
+                y: Position::Relative(-1 * (*cursor_y as isize)),
+            }])?;
+        }
+
         let osc = OperatingSystemCommand::ITermProprietary(ITermProprietary::File(Box::new(
             ITermFileData {
                 name: None,
@@ -240,17 +382,20 @@ impl ImgCatCommand {
                 data,
             },
         )));
-        println!("{}", osc);
+        println!("{begin}{osc}{end}");
 
-        if self.position.is_some() {
-            let restore_cursor = Esc::Code(EscCode::DecRestoreCursorPosition);
+        if let (Ok((_cursor_x, cursor_y)), true) = (&image_dims, needs_force_cursor_move) {
+            // tell the terminal that doesn't fully understand the image sequence
+            // to move the cursor to where it should end up
+            term.render(&[Change::CursorPosition {
+                x: Position::Absolute(0),
+                y: Position::Relative(*cursor_y as isize),
+            }])?;
+        } else if self.position.is_some() {
             print!("{restore_cursor}");
         }
 
         if self.hold {
-            let caps = Capabilities::new_from_env()?;
-            let mut term = termwiz::terminal::new_terminal(caps)?;
-
             while let Ok(Some(event)) = term.poll_input(None) {
                 match event {
                     InputEvent::Key(KeyEvent {
@@ -297,12 +442,47 @@ impl SetCwdCommand {
         let host = host.to_str().unwrap_or("localhost");
         url.set_host(Some(host))?;
 
+        let (begin, end) = self.tmux_passthru.unwrap_or_default().get();
+
         let osc = OperatingSystemCommand::CurrentWorkingDirectory(url.into());
-        print!("{}", osc);
+        print!("{begin}{osc}{end}");
+        if !begin.is_empty() {
+            // Tmux understands OSC 7 but won't automatically pass it through.
+            // <https://github.com/tmux/tmux/issues/3127#issuecomment-1076300455>
+            // Let's do it again explicitly now.
+            print!("{osc}");
+        }
         Ok(())
     }
 }
 
+#[derive(Copy, Clone, Debug, ValueEnum, Default)]
+enum TmuxPassthru {
+    Disable,
+    Enable,
+    #[default]
+    Detect,
+}
+
+impl TmuxPassthru {
+    fn is_tmux() -> bool {
+        std::env::var_os("TMUX").is_some()
+    }
+
+    fn get(&self) -> (&'static str, &'static str) {
+        let enabled = match self {
+            Self::Enable => true,
+            Self::Detect => Self::is_tmux(),
+            Self::Disable => false,
+        };
+        if enabled {
+            ("\u{1b}Ptmux;\u{1b}", "\u{1b}\\")
+        } else {
+            ("", "")
+        }
+    }
+}
+
 fn terminate_with_error_message(err: &str) -> ! {
     log::error!("{}; terminating", err);
     std::process::exit(1);