iidx, sdvx: booting fullscreen with three or more monitors (#612)

## Link to GitHub Issue or related Pull Request, if one exists
#345 

Hoping that #180 is also addressed by this change, but I can't test it.

## Description of change

IIDX (TDJ) and SDVX (Valk) will now be able to boot in full screen even
with 3 or more monitors attached.

spice will "hide" from the game any monitors other than the primary
monitor and one other monitor for subscreen, allowing it to boot without
crashing - accomplished by clearing `DISPLAY_DEVICE_ATTACHED_TO_DESKTOP`
flag when monitors are enumerated over `EnumDisplayDevicesA`, and
fortunately both games seem to respect that flag.

Add options (one for IIDX, another for SDVX) that lets you specify the
monitor ID of the second monitor.

When the option is set, user-specified monitor will be used as the
subscreen. When the option is not set, spice will just pick the
lowest-indexed monitor as the subscreen, not counting the primary one of
course.

## Testing
Tested both games with three monitors. Should probably work for four or
more.

src/spice2x/util/sysutils.cpp

@@ -10,6 +10,9 @@
 #include <windows.h>
 #undef WIN32_NO_STATUS
 
+#include "hooks/graphics/graphics.h"
+#include "avs/game.h"
+#include "util/detour.h"
 #include "util/libutils.h"
 #include "util/logging.h"
 #include "util/utils.h"
@@ -86,6 +89,9 @@ namespace sysutils {
     );
     static GetSystemFirmwareTable_t GetSystemFirmwareTable = nullptr;
 
+    std::string SECOND_MONITOR_OVERRIDE = "";
+    static decltype(EnumDisplayDevicesA) *EnumDisplayDevicesA_orig = nullptr;
+
     void print_smbios() {
         DWORD bytes_written = 0;
         DWORD table_size = 0;
@@ -216,16 +222,13 @@ namespace sysutils {
         if (adapter->StateFlags & DISPLAY_DEVICE_MIRRORING_DRIVER) {
             return;
         }
-        if (!(adapter->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP)) {
-            return;
-        }
         std::string prefix("device");
         if (is_monitor) {
             prefix = "    adapter";
         }
         log_misc("gpuinfo", "{} {} device name   : {}", prefix.c_str(), index, adapter->DeviceName);
         log_misc("gpuinfo", "{} {} device string : {}", prefix.c_str(), index, adapter->DeviceString);
-        log_dbug("gpuinfo", "{} {} flags         : 0x{:x}", prefix.c_str(), index, adapter->StateFlags);
+        log_misc("gpuinfo", "{} {} flags         : 0x{:x}", prefix.c_str(), index, adapter->StateFlags);
 
         if (!is_monitor)  {
             // get extended info for better friendly name of monitors
@@ -264,6 +267,11 @@ namespace sysutils {
                 index,
                 (adapter->StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) ? "yes" : "no");
 
+            log_misc(
+                "gpuinfo", "{} {} is attached   : {}",
+                prefix.c_str(),
+                index,
+                (adapter->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) ? "yes" : "no");
         }
     }
 
@@ -409,4 +417,119 @@ namespace sysutils {
         static const std::vector<MonitorEntry> monitors = enumerate_monitors_internal();
         return monitors;
     }
+
+    static
+    BOOL
+    __stdcall
+    EnumDisplayDevicesA_hook(
+        LPCSTR lpDevice,
+        DWORD iDevNum,
+        PDISPLAY_DEVICEA lpDisplayDevice,
+        DWORD dwFlags
+    ) {
+        if (EnumDisplayDevicesA_orig == nullptr) {
+            return false;
+        }
+
+        // caller is enumerating monitors (adapters), not devices
+        if (lpDevice != nullptr) {
+            log_misc("sysutils", "EnumDisplayDevicesA: returning original results for device {} [{}]", lpDevice, iDevNum);
+            return EnumDisplayDevicesA_orig(lpDevice, iDevNum, lpDisplayDevice, dwFlags);
+        }
+
+        // call the original first
+        const auto result_orig = EnumDisplayDevicesA_orig(nullptr, iDevNum, lpDisplayDevice, dwFlags);
+        if (!result_orig) {
+            return result_orig;
+        }
+
+        log_misc(
+            "sysutils",
+            "EnumDisplayDevicesA_orig: {} [{}], StateFlags={:#x}",
+            lpDisplayDevice->DeviceName,
+            iDevNum,
+            lpDisplayDevice->StateFlags);
+
+        // this one is not relevant
+        if ((lpDisplayDevice->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) == 0) {
+            return result_orig;
+        }
+
+        // if this is for the primary device, just return it
+        if (lpDisplayDevice->StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) {
+            log_misc(
+                "sysutils",
+                "EnumDisplayDevicesA: returning original results for primary monitor [{}], {}, result={}",
+                iDevNum,
+                lpDisplayDevice->DeviceName,
+                result_orig);
+            return result_orig;
+        }
+
+        // this is not 100% confirmed, but hope that this helps in cases where second monitors get used
+        // even though NumberOfAdaptersInGroup was set to 1 on hybrid laptops
+        if (GRAPHICS_FORCE_SINGLE_ADAPTER) {
+            lpDisplayDevice->StateFlags &= ~DISPLAY_DEVICE_ATTACHED_TO_DESKTOP;
+            log_info(
+                "sysutils",
+                "hiding this monitor, {} @ index {} (-graphics-single-adapter)",
+                lpDisplayDevice->DeviceName,
+                iDevNum);
+            return result_orig;
+        }
+
+        // for the second device (subscreen)...
+        if (SECOND_MONITOR_OVERRIDE.empty()) {
+            // if there is no user override, we find the device with lowest index (that isn't primary)
+            for (DWORD i = 0;; i++) {
+                DISPLAY_DEVICEA device = {};
+                device.cb = sizeof(device);
+                const auto result = EnumDisplayDevicesA_orig(nullptr, i, &device, 0);
+                if (!result) {
+                    break;
+                }
+                if (device.StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) {
+                    // primary, skip this
+                    continue;
+                }
+                if ((device.StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) == 0) {
+                    // disconnected, skip this as well
+                    continue;
+                }
+                if (std::string(lpDisplayDevice->DeviceName) == device.DeviceName) {
+                    log_info(
+                        "sysutils",
+                        "returning second monitor {} @ index {}",
+                        lpDisplayDevice->DeviceName,
+                        iDevNum);
+                    return result_orig;
+                } else {
+                    // otherwise, fall through and hide this monitor
+                    break;
+                }
+            }
+        } else if (SECOND_MONITOR_OVERRIDE == lpDisplayDevice->DeviceName) {
+            // if there is one preferred by the user, use this one, and hide others
+            log_info(
+                "sysutils",
+                "returning second monitor, {} @ index {} (-sysutilssubmonitor)",
+                lpDisplayDevice->DeviceName,
+                iDevNum);
+            return result_orig;
+        }
+
+        // this device should not be used; pretend that it's not connected
+        log_info(
+            "sysutils",
+            "hiding this monitor, {} @ index {}",
+            lpDisplayDevice->DeviceName,
+            iDevNum);
+        lpDisplayDevice->StateFlags &= ~DISPLAY_DEVICE_ATTACHED_TO_DESKTOP;
+        return result_orig;
+    }
+
+    void hook_EnumDisplayDevicesA() {
+        EnumDisplayDevicesA_orig = detour::iat_try(
+             "EnumDisplayDevicesA", EnumDisplayDevicesA_hook, avs::game::DLL_INSTANCE);
+    }
 }