[newlib-cygwin] Cygwin: console: Store console mode only when console is opened

[Takashi Yano]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=09ae9f6ee99e2b58dcb17af563f15e8026b773ae

commit 09ae9f6ee99e2b58dcb17af563f15e8026b773ae
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Wed May 7 15:40:22 2025 +0900

    Cygwin: console: Store console mode only when console is opened
    
    ... and restore it when app exits. The commit 0bfd91d57863 has a bug
    that the console mode is stored into the shared memory when both:
      (1) cygwin process is started from non-cygwin process.
      (2) cygwin process started from non-cygwin process exits.
    (1) is intended, but (2) is not. Due to (2), the stored console mode
    is unexpectedly broken when the cygwin process exits. Then the mode
    restored will be not as expected. This causes undesired console mode
    in the use case that cygwin and non-cygwin apps are mixed.
    
    With this patch, the console mode will stored only in the case (1).
    This is done by putting the code, which stores the console mode, into
    fhandler_console::open() rather than fhandler_console::set_input_mode()
    and fhandler_console::set_output_mode().
    
    Fixes: 0bfd91d57863 ("Cygwin: console: tty::restore really restores the previous mode")
    Reported-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
    Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>

Diff:
---
 winsup/cygwin/fhandler/console.cc | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/winsup/cygwin/fhandler/console.cc b/winsup/cygwin/fhandler/console.cc
index 2e19e0dd7..16352d04d 100644
--- a/winsup/cygwin/fhandler/console.cc
+++ b/winsup/cygwin/fhandler/console.cc
@@ -771,6 +771,8 @@ fhandler_console::setup ()
       con.disable_master_thread = true;
       con.master_thread_suspended = false;
       con.num_processed = 0;
+      con.curr_input_mode = tty::restore;
+      con.curr_output_mode = tty::restore;
     }
 }
 
@@ -849,11 +851,6 @@ fhandler_console::set_input_mode (tty::cons_mode m, const termios *t,
 	flags |= ENABLE_PROCESSED_INPUT;
       break;
     }
-  if (con.curr_input_mode != tty::cygwin && m == tty::cygwin)
-    {
-      prev_input_mode_backup = con.prev_input_mode;
-      con.prev_input_mode = oflags;
-    }
   con.curr_input_mode = m;
   SetConsoleMode (p->input_handle, flags);
   if (!(oflags & ENABLE_VIRTUAL_TERMINAL_INPUT)
@@ -893,11 +890,6 @@ fhandler_console::set_output_mode (tty::cons_mode m, const termios *t,
 	flags |= DISABLE_NEWLINE_AUTO_RETURN;
       break;
     }
-  if (con.curr_output_mode != tty::cygwin && m == tty::cygwin)
-    {
-      prev_output_mode_backup = con.prev_output_mode;
-      GetConsoleMode (p->output_handle, &con.prev_output_mode);
-    }
   con.curr_output_mode = m;
   acquire_attach_mutex (mutex_timeout);
   DWORD resume_pid = attach_console (con.owner);
@@ -1845,6 +1837,12 @@ fhandler_console::open (int flags, mode_t)
   handle_set.output_handle = h;
   release_output_mutex ();
 
+  if (con.owner == GetCurrentProcessId ())
+    {
+      GetConsoleMode (get_handle (), &con.prev_input_mode);
+      GetConsoleMode (get_output_handle (), &con.prev_output_mode);
+    }
+
   wpbuf.init ();
 
   handle_set.input_mutex = input_mutex;
@@ -1890,6 +1888,19 @@ fhandler_console::open (int flags, mode_t)
 	setenv ("TERM", "cygwin", 1);
     }
 
+  if (con.curr_input_mode != tty::cygwin)
+    {
+      prev_input_mode_backup = con.prev_input_mode;
+      GetConsoleMode (get_handle (), &con.prev_input_mode);
+      set_input_mode (tty::cygwin, &get_ttyp ()->ti, &handle_set);
+    }
+  if (con.curr_output_mode != tty::cygwin)
+    {
+      prev_output_mode_backup = con.prev_output_mode;
+      GetConsoleMode (get_output_handle (), &con.prev_output_mode);
+      set_output_mode (tty::cygwin, &get_ttyp ()->ti, &handle_set);
+    }
+
   debug_printf ("opened conin$ %p, conout$ %p", get_handle (),
 		get_output_handle ());
 
@@ -4738,7 +4749,7 @@ fhandler_console::cons_mode_on_close (handle_set_t *p)
   NTSTATUS status =
     NtQueryInformationProcess (GetCurrentProcess (), ProcessBasicInformation,
 			       &pbi, sizeof (pbi), NULL);
-  if (NT_SUCCESS (status)
+  if (NT_SUCCESS (status) && cygwin_pid (con.owner)
       && !process_alive ((DWORD) pbi.InheritedFromUniqueProcessId))
     /* Execed from normal cygwin process and the parent has been exited. */
     return tty::cygwin;