win: add ENABLE_VIRTUAL_TERMINAL_INPUT raw tty mode (#4688)

Windows provides the `ENABLE_VIRTUAL_TERMINAL_INPUT` flag for TTY input
streams as a companion flag to `ENABLE_VIRTUAL_TERMINAL_PROCESSING`,
which libuv is already setting for TTY output streams.

Setting this flag lets the terminal emulator perform some of the
processing that libuv already currently does for input events,
but most notably enables receiving control sequences that are
otherwise entirely unavailable, e.g. for bracketed paste
(which the Node.js readline implementation added basic support for
in https://github.com/nodejs/node/commit/87af913b66eab78088acfd).

libuv currently already provides translations for key events to
control sequences, i.e. what this mode is intended to provide,
but libuv does not and cannot translate all such events.
Since the control sequences differ from the ones that Windows
has chosen to standardize on, and applications may not be expecting
this change, this is opt-in for now (but ideally will be the default
behavior starting in libuv v2.x, should that ever happen).

Another downside of this change is that not all shells reset
this mode when an application exits. For example, when running a
Node.js program with this flag enabled inside of PowerShell in
Windows terminal, if the application exits while in raw TTY input mode,
neither the shell nor the terminal emulator reset this flag, rendering
the input stream unusable.

While there's general awareness of the problem that console state is
global state rather than per-process (same as on UNIX platforms),
it seems that applications like PowerShell aren't expecting to need to
unset this flag on the input stream, only its output counterpart
(e.g. https://github.com/PowerShell/PowerShell/blob/4e7942135f998ab40fd3ae298b020e161a76d4ef/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs#L1156).

Hence, `uv_tty_reset_mode()` is extended to reset the terminal
to its original state if the new mode is being used.

Refs: https://github.com/nodejs/node/commit/87af913b66eab78088acfd7f3b57d35e22c5f1ba
Refs: https://github.com/microsoft/terminal/issues/4954

3 files changed, 71 insertions, 22 deletions

diff --git a/src/unix/tty.c b/src/unix/tty.c
index 793054ba..b8610720 100644
--- a/src/unix/tty.c
+++ b/src/unix/tty.c
@@ -284,6 +284,11 @@ int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) {
   int fd;
   int rc;
 
+  if (uv__is_raw_tty_mode(mode)) {
+    /* There is only a single raw TTY mode on UNIX. */
+    mode = UV_TTY_MODE_RAW;
+  }
+
   if (tty->mode == (int) mode)
     return 0;
 
@@ -324,6 +329,8 @@ int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) {
     case UV_TTY_MODE_IO:
       uv__tty_make_raw(&tmp);
       break;
+    default:
+      UNREACHABLE();
   }
 
   /* Apply changes after draining */
diff --git a/src/uv-common.h b/src/uv-common.h
index 8e779ba5..b9a8e976 100644
--- a/src/uv-common.h
+++ b/src/uv-common.h
@@ -126,7 +126,7 @@ enum {
 
   /* Only used by uv_tty_t handles. */
   UV_HANDLE_TTY_READABLE                = 0x01000000,
-  UV_HANDLE_TTY_RAW                     = 0x02000000,
+  UV_HANDLE_UNUSED0                     = 0x02000000,
   UV_HANDLE_TTY_SAVED_POSITION          = 0x04000000,
   UV_HANDLE_TTY_SAVED_ATTRIBUTES        = 0x08000000,
 
@@ -141,6 +141,10 @@ enum {
   UV_HANDLE_REAP                        = 0x10000000
 };
 
+static inline int uv__is_raw_tty_mode(uv_tty_mode_t m) {
+  return m == UV_TTY_MODE_RAW || m == UV_TTY_MODE_RAW_VT;
+}
+
 int uv__loop_configure(uv_loop_t* loop, uv_loop_option option, va_list ap);
 
 void uv__loop_close(uv_loop_t* loop);
diff --git a/src/win/tty.c b/src/win/tty.c
index c0339ded..66ca99cd 100644
--- a/src/win/tty.c
+++ b/src/win/tty.c
@@ -58,6 +58,9 @@
 #ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
 #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
 #endif
+#ifndef ENABLE_VIRTUAL_TERMINAL_INPUT
+#define ENABLE_VIRTUAL_TERMINAL_INPUT 0x0200
+#endif
 
 #define CURSOR_SIZE_SMALL     25
 #define CURSOR_SIZE_LARGE     100
@@ -119,7 +122,10 @@ static int uv_tty_virtual_width = -1;
  * handle signalling SIGWINCH
  */
 
-static HANDLE uv__tty_console_handle = INVALID_HANDLE_VALUE;
+static HANDLE uv__tty_console_handle_out = INVALID_HANDLE_VALUE;
+static HANDLE uv__tty_console_handle_in = INVALID_HANDLE_VALUE;
+static DWORD uv__tty_console_in_original_mode = (DWORD)-1;
+static volatile LONG uv__tty_console_in_need_mode_reset = 0;
 static int uv__tty_console_height = -1;
 static int uv__tty_console_width = -1;
 static HANDLE uv__tty_console_resized = INVALID_HANDLE_VALUE;
@@ -159,19 +165,21 @@ static uv_tty_vtermstate_t uv__vterm_state = UV_TTY_UNSUPPORTED;
 static void uv__determine_vterm_state(HANDLE handle);
 
 void uv__console_init(void) {
+  DWORD dwMode;
+
   if (uv_sem_init(&uv_tty_output_lock, 1))
     abort();
-  uv__tty_console_handle = CreateFileW(L"CONOUT$",
-                                       GENERIC_READ | GENERIC_WRITE,
-                                       FILE_SHARE_WRITE,
-                                       0,
-                                       OPEN_EXISTING,
-                                       0,
-                                       0);
-  if (uv__tty_console_handle != INVALID_HANDLE_VALUE) {
+  uv__tty_console_handle_out = CreateFileW(L"CONOUT$",
+                                           GENERIC_READ | GENERIC_WRITE,
+                                           FILE_SHARE_WRITE,
+                                           0,
+                                           OPEN_EXISTING,
+                                           0,
+                                           0);
+  if (uv__tty_console_handle_out != INVALID_HANDLE_VALUE) {
     CONSOLE_SCREEN_BUFFER_INFO sb_info;
     uv_mutex_init(&uv__tty_console_resize_mutex);
-    if (GetConsoleScreenBufferInfo(uv__tty_console_handle, &sb_info)) {
+    if (GetConsoleScreenBufferInfo(uv__tty_console_handle_out, &sb_info)) {
       uv__tty_console_width = sb_info.dwSize.X;
       uv__tty_console_height = sb_info.srWindow.Bottom - sb_info.srWindow.Top + 1;
     }
@@ -179,6 +187,18 @@ void uv__console_init(void) {
                       NULL,
                       WT_EXECUTELONGFUNCTION);
   }
+  uv__tty_console_handle_in = CreateFileW(L"CONIN$",
+                                          GENERIC_READ | GENERIC_WRITE,
+                                          FILE_SHARE_READ,
+                                          0,
+                                          OPEN_EXISTING,
+                                          0,
+                                          0);
+  if (uv__tty_console_handle_in != INVALID_HANDLE_VALUE) {
+    if (GetConsoleMode(uv__tty_console_handle_in, &dwMode)) {
+      uv__tty_console_in_original_mode = dwMode;
+    }
+  }
 }
 
 
@@ -253,7 +273,9 @@ int uv_tty_init(uv_loop_t* loop, uv_tty_t* tty, uv_file fd, int unused) {
     /* Initialize TTY input specific fields. */
     tty->flags |= UV_HANDLE_TTY_READABLE | UV_HANDLE_READABLE;
     /* TODO: remove me in v2.x. */
-    tty->tty.rd.unused_ = NULL;
+    tty->tty.rd.mode.unused_ = NULL;
+    /* Partially overwrites unused_ again. */
+    tty->tty.rd.mode.mode = 0;
     tty->tty.rd.read_line_buffer = uv_null_buf_;
     tty->tty.rd.read_raw_wait = NULL;
 
@@ -344,6 +366,7 @@ static void uv__tty_capture_initial_style(
 
 int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) {
   DWORD flags;
+  DWORD try_set_flags;
   unsigned char was_reading;
   uv_alloc_cb alloc_cb;
   uv_read_cb read_cb;
@@ -353,14 +376,19 @@ int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) {
     return UV_EINVAL;
   }
 
-  if (!!mode == !!(tty->flags & UV_HANDLE_TTY_RAW)) {
+  if ((int)mode == tty->tty.rd.mode.mode) {
     return 0;
   }
 
+  try_set_flags = 0;
   switch (mode) {
     case UV_TTY_MODE_NORMAL:
       flags = ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT;
       break;
+    case UV_TTY_MODE_RAW_VT:
+      try_set_flags = ENABLE_VIRTUAL_TERMINAL_INPUT;
+      InterlockedExchange(&uv__tty_console_in_need_mode_reset, 1);
+      /* fallthrough */
     case UV_TTY_MODE_RAW:
       flags = ENABLE_WINDOW_INPUT;
       break;
@@ -386,16 +414,16 @@ int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) {
   }
 
   uv_sem_wait(&uv_tty_output_lock);
-  if (!SetConsoleMode(tty->handle, flags)) {
+  if (!SetConsoleMode(tty->handle, flags | try_set_flags) &&
+      !SetConsoleMode(tty->handle, flags)) {
     err = uv_translate_sys_error(GetLastError());
     uv_sem_post(&uv_tty_output_lock);
     return err;
   }
   uv_sem_post(&uv_tty_output_lock);
 
-  /* Update flag. */
-  tty->flags &= ~UV_HANDLE_TTY_RAW;
-  tty->flags |= mode ? UV_HANDLE_TTY_RAW : 0;
+  /* Update mode. */
+  tty->tty.rd.mode.mode = mode;
 
   /* If we just stopped reading, restart. */
   if (was_reading) {
@@ -614,7 +642,7 @@ static void uv__tty_queue_read_line(uv_loop_t* loop, uv_tty_t* handle) {
 
 
 static void uv__tty_queue_read(uv_loop_t* loop, uv_tty_t* handle) {
-  if (handle->flags & UV_HANDLE_TTY_RAW) {
+  if (uv__is_raw_tty_mode(handle->tty.rd.mode.mode)) {
     uv__tty_queue_read_raw(loop, handle);
   } else {
     uv__tty_queue_read_line(loop, handle);
@@ -702,7 +730,7 @@ void uv_process_tty_read_raw_req(uv_loop_t* loop, uv_tty_t* handle,
   handle->flags &= ~UV_HANDLE_READ_PENDING;
 
   if (!(handle->flags & UV_HANDLE_READING) ||
-      !(handle->flags & UV_HANDLE_TTY_RAW)) {
+      !(uv__is_raw_tty_mode(handle->tty.rd.mode.mode))) {
     goto out;
   }
 
@@ -1050,7 +1078,7 @@ int uv__tty_read_stop(uv_tty_t* handle) {
   if (!(handle->flags & UV_HANDLE_READ_PENDING))
     return 0;
 
-  if (handle->flags & UV_HANDLE_TTY_RAW) {
+  if (uv__is_raw_tty_mode(handle->tty.rd.mode.mode)) {
     /* Cancel raw read. Write some bullshit event to force the console wait to
      * return. */
     memset(&record, 0, sizeof record);
@@ -2293,7 +2321,17 @@ void uv__tty_endgame(uv_loop_t* loop, uv_tty_t* handle) {
 
 
 int uv_tty_reset_mode(void) {
-  /* Not necessary to do anything. */
+  /**
+   * Shells on Windows do know to reset output flags after a program exits,
+   * but not necessarily input flags, so we do that for them.
+   */
+  if (
+    uv__tty_console_handle_in != INVALID_HANDLE_VALUE &&
+    uv__tty_console_in_original_mode != (DWORD)-1 &&
+    InterlockedExchange(&uv__tty_console_in_need_mode_reset, 0) != 0
+  ) {
+    SetConsoleMode(uv__tty_console_handle_in, uv__tty_console_in_original_mode);
+  }
   return 0;
 }
 
@@ -2390,7 +2428,7 @@ static void uv__tty_console_signal_resize(void) {
   CONSOLE_SCREEN_BUFFER_INFO sb_info;
   int width, height;
 
-  if (!GetConsoleScreenBufferInfo(uv__tty_console_handle, &sb_info))
+  if (!GetConsoleScreenBufferInfo(uv__tty_console_handle_out, &sb_info))
     return;
 
   width = sb_info.dwSize.X;