[windows] file_write: require a console, not just a character device

Changes.md

@@ -16,6 +16,7 @@
 * Updated sqlite to 3.40.1
 * Updated zlib to 1.2.13
 
+* Fixed crash writing to the null device on Windows
 * Fixed SSL socket non blocking handshake throwing an exception on 64bit Windows
 * Fixed Windows 64bit architecture detection
 * Fixed critial error handler returning the wrong callstack

src/hx/libs/std/File.cpp

@@ -149,7 +149,10 @@ int _hx_std_file_write( Dynamic handle, Array<unsigned char> s, int p, int n )
 
    hx::AutoGCFreeZone zone;
 #ifdef HX_WINDOWS
-   if (_isatty(_fileno(f->io))) {
+   // _isatty is true for ANY character device - NUL, a serial port, a printer - and none of
+   // those accept WriteConsoleW. Only a real console has a console mode.
+   DWORD console_mode;
+   if (_isatty(_fileno(f->io)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(f->io)), &console_mode)) {
       fflush(f->io);
       HANDLE win_handle = (HANDLE)_get_osfhandle(_fileno(f->io));
       static const int MAX_BUFFER_SIZE = 8192;

test/regression/NulDeviceWrite/Main.hx

@@ -0,0 +1,13 @@
+// The null device is a character device, so _isatty reports it as a tty. file_write used that as
+// its test for a console and called WriteConsoleW, which only accepts a console handle; the failure
+// threw from inside a GC-free zone and took the process down instead of reporting an error.
+function main() {
+	final nul = Sys.systemName() == "Windows" ? "NUL" : "/dev/null";
+
+	final out = sys.io.File.write(nul);
+	out.writeString("written to the null device\n");
+	out.flush();
+	out.close();
+
+	Sys.println("ok");
+}

test/regression/NulDeviceWrite/build.hxml

@@ -0,0 +1,2 @@
+-cpp bin
+-m Main

test/regression/NulDeviceWrite/stdout.txt

@@ -0,0 +1 @@
+ok