Improved unicode wcwidth support

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/unicode/google-libapps"]
+	path = src/unicode/google-libapps
+	url = https://chromium.googlesource.com/apps/libapps

configure.ac

@@ -508,6 +508,7 @@ AC_CONFIG_FILES([
   src/protobufs/Makefile
   src/statesync/Makefile
   src/terminal/Makefile
+  src/unicode/Makefile
   src/util/Makefile
   scripts/Makefile
   src/examples/Makefile

man/mosh-client.1

@@ -20,7 +20,7 @@ mosh-client \- client-side helper for mosh
 .SH SYNOPSIS
 MOSH_KEY=KEY
 .B mosh-client 
-[\-v]
+[\-v] [\-w fd] [\-W]
 IP PORT
 .br
 .B mosh-client 
@@ -55,6 +55,12 @@ error.  More instances of this flag will result in more debugging
 information.  If standard error is not redirected from the terminal,
 the display will be corrupted and quickly become unusable.
 
+The \-w fd option specifies a character width table for terminal emulation.  See
+.BR mosh (1).
+
+The \-W option causes \fBmosh-server\fP to treat all Unicode East
+Asian ambiguous-width characters as wide.
+
 .SH ENVIRONMENT VARIABLES
 
 .TP

man/mosh.1

@@ -262,6 +262,24 @@ This option ignores any configuration in
 .B ssh_config
 for the same hostname.
 
+.TP
+.B \-\-widths\-file=\fIFILENAME\fP
+If this option is given, Mosh will load a character width table from
+the given file.  Otherwise, it defaults to "~/.mosh-widths".  This
+feature is currently under development and underdocumented.
+
+.TP
+.B \-\-no\-widths\-file
+If this option is given, Mosh will not load any character width table
+file, and will instead use its built-in default table.
+
+.TP
+.B \-\-eaw\-wide
+.B \-\-no\-eaw\-wide
+Enable/disable interpretation of East Asian ambiguous-width characters
+as wide.  If this option is not given, Mosh will automatically enable
+this option for Chinese, Japanese, and Korean locales.
+
 .SH ESCAPE SEQUENCES
 
 The default escape character used by Mosh is ASCII RS (decimal 30).

scripts/mosh.pl

@@ -34,7 +34,9 @@
 
 use warnings;
 use strict;
+use Fcntl;
 use Getopt::Long;
+use IO::Handle;
 use IO::Socket;
 use Text::ParseWords;
 use Socket qw(IPPROTO_TCP);
@@ -72,6 +74,12 @@ BEGIN
 
 my $use_remote_ip = 'proxy';
 
+my $no_widths_file = 0;
+my $default_widths_file = "$ENV{HOME}/.mosh-widths"; # XXX need better defined location
+my $widths_file = $default_widths_file;
+
+my $eaw_wide = undef;
+
 my $family = 'prefer-inet';
 my $port_request = undef;
 
@@ -128,6 +136,17 @@ BEGIN
                              discovering the remote IP address to use for mosh
                              (default: "proxy")
 
+        --widths-file=filename  Read a character width table from the specified
+                             filename (default: "~/.mosh-widths")
+
+        --no-widths-file     Do not read the default width table file
+
+        --eaw-wide           Treat Unicode East Asian ambiguous width characters
+                             as wide (default: depends on locale)
+
+        --no-eaw-wide        Treat Unicode East Asian ambiguous width characters
+                             as narrow (default: depends on locale)
+
         --help               this message
         --version            version and copyright information
 
@@ -171,7 +190,11 @@ sub predict_check {
 	    'version' => \$version,
 	    'fake-proxy!' => \my $fake_proxy,
 	    'bind-server=s' => \$bind_ip,
-	    'experimental-remote-ip=s' => \$use_remote_ip) or die $usage;
+	    'experimental-remote-ip=s' => \$use_remote_ip,
+	    'widths-file=s' => \$widths_file,
+	    'no-widths-file' => \$no_widths_file,
+	    'eaw-wide!' => \$eaw_wide,
+    ) or die $usage;
 
 if ( defined $help ) {
     print $usage;
@@ -233,6 +256,18 @@ sub predict_check {
   }
 }
 
+
+if ( $no_widths_file ) {
+    undef $widths_file;
+}
+
+if ( defined $widths_file && ! -f $widths_file ) {
+    if ( $widths_file ne $default_widths_file ) {
+	warn "$0: no width file found at $widths_file\n";
+    }
+    undef $widths_file;
+}
+
 delete $ENV{ 'MOSH_PREDICTION_DISPLAY' };
 
 my $userhost;
@@ -327,6 +362,11 @@ sub predict_check {
   $colors = 0;
 }
 
+if ( !defined( $eaw_wide ) ) {
+    $eaw_wide = (($ENV{LC_ALL} || $ENV{LC_CTYPE} || $ENV{LANG}) =~ /^(?:ja|ko|zh)(?:_|$)/);
+}
+
+
 $ENV{ 'MOSH_CLIENT_PID' } = $$; # We don't support this, but it's useful for test and debug.
 
 # If we are using a locally-resolved address, we have to get it before we fork,
@@ -462,7 +502,17 @@ sub predict_check {
   $ENV{ 'MOSH_KEY' } = $key;
   $ENV{ 'MOSH_PREDICTION_DISPLAY' } = $predict;
   $ENV{ 'MOSH_NO_TERM_INIT' } = '1' if !$term_init;
-  exec {$client} ("$client", "-# @cmdline |", $ip, $port);
+  my @exec_argv = ( "$client", "-# @cmdline |" );
+  if ( defined $widths_file ) {
+      my $wfd = POSIX::open( $widths_file, O_RDONLY )
+	  or die "$0: Could not open character widths file $widths_file";
+      push @exec_argv, ('-w', $wfd);
+  }
+  if ( $eaw_wide ) {
+      push @exec_argv, '-W';
+  }
+  push @exec_argv, ($ip, $port);
+  exec {$client} @exec_argv;
 }
 
 sub shell_quote { join ' ', map {(my $a = $_) =~ s/'/'\\''/g; "'$a'"} @_ }

src/Makefile.am

@@ -1 +1 @@
-SUBDIRS = protobufs util crypto terminal network statesync frontend examples tests
+SUBDIRS = protobufs util crypto terminal network statesync frontend examples tests unicode

src/examples/Makefile.am

@@ -15,11 +15,11 @@ decrypt_LDADD = ../crypto/libmoshcrypto.a $(CRYPTO_LIBS)
 
 parse_SOURCES = parse.cc
 parse_CPPFLAGS = -I$(srcdir)/../terminal -I$(srcdir)/../util
-parse_LDADD = ../terminal/libmoshterminal.a ../util/libmoshutil.a
+parse_LDADD = ../terminal/libmoshterminal.a ../util/libmoshutil.a ../network/libmoshnetwork.a
 
 termemu_SOURCES = termemu.cc
 termemu_CPPFLAGS = -I$(srcdir)/../terminal -I$(srcdir)/../util -I$(srcdir)/../statesync -I../protobufs
-termemu_LDADD = ../terminal/libmoshterminal.a ../util/libmoshutil.a ../statesync/libmoshstatesync.a ../protobufs/libmoshprotos.a $(TINFO_LIBS) $(protobuf_LIBS)
+termemu_LDADD = ../terminal/libmoshterminal.a ../util/libmoshutil.a ../statesync/libmoshstatesync.a ../protobufs/libmoshprotos.a ../network/libmoshnetwork.a $(TINFO_LIBS) $(protobuf_LIBS)
 
 ntester_SOURCES = ntester.cc
 ntester_CPPFLAGS = -I$(srcdir)/../util -I$(srcdir)/../statesync -I$(srcdir)/../terminal -I$(srcdir)/../network -I$(srcdir)/../crypto -I../protobufs $(protobuf_CFLAGS)

src/examples/benchmark.cc

@@ -84,12 +84,14 @@ int main( int argc, char **argv )
 	exit(1);
       }
     }
-    Framebuffer local_framebuffers[ 2 ] = { Framebuffer(width,height), Framebuffer(width,height) };
+    ChWidthPtr widths = shared::make_shared<ChWidth>();
+    widths->apply_diff( ChWidth::get_default() );
+    Framebuffer local_framebuffers[ 2 ] = { Framebuffer( width, height, widths ), Framebuffer( width, height, widths ) };
     Framebuffer *local_framebuffer = &(local_framebuffers[ fbmod ]);
     Framebuffer *new_state = &(local_framebuffers[ !fbmod ]);
     Overlay::OverlayManager overlays;
     Display display( true );
-    Complete local_terminal( width, height );
+    Complete local_terminal( width, height, widths );
 
     /* Adopt native locale */
     set_native_locale();

src/examples/termemu.cc

@@ -232,8 +232,10 @@ static void emulate_terminal( int fd )
   }
 
   /* open parser and terminal */
-  Terminal::Complete complete( window_size.ws_col, window_size.ws_row );
-  Terminal::Framebuffer state( window_size.ws_col, window_size.ws_row );
+  ChWidthPtr widths = shared::make_shared<ChWidth>();
+  widths->apply_diff( ChWidth::get_default() );
+  Terminal::Complete complete( window_size.ws_col, window_size.ws_row, widths );
+  Terminal::Framebuffer state( window_size.ws_col, window_size.ws_row, widths );
 
   /* open display */
   Terminal::Display display( true ); /* use TERM to initialize */

src/frontend/mosh-client.cc

@@ -33,10 +33,12 @@
 #include "config.h"
 #include "version.h"
 
+#include <err.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "stmclient.h"
+#include "chwidth.h"
 #include "crypto.h"
 #include "locale_utils.h"
 #include "fatal_assert.h"
@@ -82,7 +84,7 @@ static void print_version( FILE *file )
 static void print_usage( FILE *file, const char *argv0 )
 {
   print_version( file );
-  fprintf( file, "\nUsage: %s [-# 'ARGS'] IP PORT\n"
+  fprintf( file, "\nUsage: %s [-v] [-w fd] [-W] [-# 'ARGS'] IP PORT\n"
 	   "       %s -c\n", argv0, argv0 );
 }
 
@@ -108,6 +110,8 @@ int main( int argc, char *argv[] )
 #endif
 {
   unsigned int verbose = 0;
+  int chwidth_fd = -1;
+  bool eaw_wide = false;
   /* For security, make sure we don't dump core */
   Crypto::disable_dumping_core();
 
@@ -127,7 +131,7 @@ int main( int argc, char *argv[] )
   }
 
   int opt;
-  while ( (opt = getopt( argc, argv, "#:cv" )) != -1 ) {
+  while ( (opt = getopt( argc, argv, "#:cvw:W" )) != -1 ) {
     switch ( opt ) {
     case '#':
       // Ignore the original arguments to mosh wrapper
@@ -139,6 +143,12 @@ int main( int argc, char *argv[] )
     case 'v':
       verbose++;
       break;
+    case 'w':
+      chwidth_fd = atoi( optarg );
+      break;
+    case 'W':
+      eaw_wide = true;
+      break;
     default:
       print_usage( stderr, argv[ 0 ] );
       exit( 1 );
@@ -189,9 +199,49 @@ int main( int argc, char *argv[] )
   /* Adopt native locale */
   set_native_locale();
 
+  /* Install our default character width table. */
+  ChWidthPtr chwidths = shared::make_shared<ChWidth>();
+  if ( !chwidths->apply_diff( ChWidth::get_default() )) {
+    errx( 1, "bad default table" );
+  }
+
+  /* Load and apply a user character width table if provided. */
+  std::string user_map;
+  if ( chwidth_fd != -1 ) {
+    /* Read character width table. */
+    /* Dup it to see if it's actually open. */
+    int chwidth_fd2 = dup( chwidth_fd );
+    if ( chwidth_fd2 == -1 ) {
+      err( 1, "dup chwidth fd" );
+    }
+    close( chwidth_fd2 );
+    /* C++ streams offers no standard way to attach to an fd.  So use stdio. */
+    FILE *chwidth_file = fdopen( chwidth_fd, "r" );
+    if ( !chwidth_file ) {
+      err( 1, "fdopen chwidth fd" );
+    }
+    /* Read characters. */
+    user_map.reserve( 65536 );
+    int ch;
+    while ( EOF != ( ch = getc_unlocked( chwidth_file ) ) ) {
+      user_map.push_back( static_cast<char>( ch ) );
+    }
+    fclose( chwidth_file );
+    /* Apply changes to default table.  This may replace it entirely. */
+    if ( !chwidths->apply_diff( user_map )) {
+      errx( 1, "bad user character widths table" );
+    }
+  }
+  /* Apply East Asian width-wide delta if requested. */
+  if ( eaw_wide ) {
+    if ( !chwidths->apply_diff( ChWidth::get_eaw_delta() )) {
+      errx( 1, "bad delta table" );
+    }
+  }
+
   bool success = false;
   try {
-    STMClient client( ip, desired_port, key.c_str(), predict_mode, verbose, predict_overwrite );
+    STMClient client( ip, desired_port, key.c_str(), predict_mode, verbose, predict_overwrite, chwidths );
     client.init();
 
     try {

src/frontend/mosh-server.cc

@@ -88,6 +88,7 @@
 #include "select.h"
 #include "timestamp.h"
 #include "fatal_assert.h"
+#include "chwidth.h"
 
 #ifndef _PATH_BSHELL
 #define _PATH_BSHELL "/bin/sh"
@@ -420,7 +421,9 @@ static int run_server( const char *desired_ip, const char *desired_port,
   }
 
   /* open parser and terminal */
-  Terminal::Complete terminal( window_size.ws_col, window_size.ws_row );
+  ChWidthPtr widths = shared::make_shared<ChWidth>();
+  widths->apply_diff( ChWidth::get_reference() );
+  Terminal::Complete terminal( window_size.ws_col, window_size.ws_row, widths );
 
   /* open network */
   Network::UserStream blank;
@@ -731,7 +734,6 @@ static void serve( int host_fd, Terminal::Complete &terminal, ServerConnection &
 	if ( network.get_remote_state_num() != last_remote_num ) {
 	  last_remote_num = network.get_remote_state_num();
 
-
 	  Network::UserStream us;
 	  us.apply_string( network.get_remote_diff() );
 	  /* apply userstream to terminal */
@@ -819,7 +821,7 @@ static void serve( int host_fd, Terminal::Complete &terminal, ServerConnection &
 	  #endif
 
 	  /* Tell child to start login session. */
-	  if ( !child_released ) {
+	  if ( !child_released && !terminal.get_hold_session() ) {
 	    if ( swrite( host_fd, "\n", 1 ) < 0) {
 	      err( 1, "child release" );
 	    }

src/frontend/stmclient.cc

@@ -51,6 +51,7 @@
 #include <util.h>
 #endif
 
+#include "compressor.h"
 #include "stmclient.h"
 #include "swrite.h"
 #include "completeterminal.h"
@@ -242,23 +243,36 @@ void STMClient::main_init( void )
   }  
 
   /* local state */
-  local_framebuffer = Terminal::Framebuffer( window_size.ws_col, window_size.ws_row );
-  new_state = Terminal::Framebuffer( 1, 1 );
+  local_framebuffer = Terminal::Framebuffer( window_size.ws_col, window_size.ws_row, chwidth_table );
+  new_state = Terminal::Framebuffer( 1, 1, chwidth_table );
 
   /* initialize screen */
   string init = display.new_frame( false, local_framebuffer, local_framebuffer );
   swrite( STDOUT_FILENO, init.data(), init.size() );
 
   /* open network */
   Network::UserStream blank;
-  Terminal::Complete local_terminal( window_size.ws_col, window_size.ws_row );
+  Terminal::Complete local_terminal( window_size.ws_col, window_size.ws_row, chwidth_table );
   network = NetworkPointer( new NetworkType( blank, local_terminal, key.c_str(), ip.c_str(), port.c_str() ) );
 
   network->set_send_delay( 1 ); /* minimal delay on outgoing keystrokes */
 
+  /* Hold session until further notice */
+  network->get_current_state().push_back( Parser::HoldSession( true ) );
   /* tell server the size of the terminal */
   network->get_current_state().push_back( Parser::Resize( window_size.ws_col, window_size.ws_row ) );
 
+  /* Now generate the network overlay for the server (generated from the fixed base and the working table) */
+  ChWidth reference_widths;
+  reference_widths.apply_diff( ChWidth::get_reference() );
+  string network_overlay = reference_widths.make_diff( *chwidth_table );
+  /* send server the wcwidth overlay */
+  network->get_current_state().
+    push_back( Parser::ChWidthOverlay( network_overlay ) );
+  /* Release session.  (In a future version of Mosh, this might be in a later message.) */
+  network->get_current_state().push_back( Parser::HoldSession( false ) );
+
+
   /* be noisy as necessary */
   network->set_verbose( verbose );
   Select::set_verbose( verbose );