iidx, sdvx: booting fullscreen with three or more monitors (#612)

## Link to GitHub Issue or related Pull Request, if one exists
#345 

Hoping that #180 is also addressed by this change, but I can't test it.

## Description of change

IIDX (TDJ) and SDVX (Valk) will now be able to boot in full screen even
with 3 or more monitors attached.

spice will "hide" from the game any monitors other than the primary
monitor and one other monitor for subscreen, allowing it to boot without
crashing - accomplished by clearing `DISPLAY_DEVICE_ATTACHED_TO_DESKTOP`
flag when monitors are enumerated over `EnumDisplayDevicesA`, and
fortunately both games seem to respect that flag.

Add options (one for IIDX, another for SDVX) that lets you specify the
monitor ID of the second monitor.

When the option is set, user-specified monitor will be used as the
subscreen. When the option is not set, spice will just pick the
lowest-indexed monitor as the subscreen, not counting the primary one of
course.

## Testing
Tested both games with three monitors. Should probably work for four or
more.

src/spice2x/games/iidx/iidx.cpp

@@ -30,6 +30,7 @@
 #include "util/memutils.h"
 #include "util/sigscan.h"
 #include "util/socd_cleaner.h"
+#include "util/sysutils.h"
 #include "util/time.h"
 #include "util/utils.h"
 #include "launcher/signal.h"
@@ -423,6 +424,14 @@ namespace games::iidx {
             init_legacy_camera_hook(FLIP_CAMS);
         }
 
+#if SPICE64
+
+        if (TDJ_MODE) {
+            sysutils::hook_EnumDisplayDevicesA();
+        }
+
+#endif
+
         // init cfgmgr32 hooks
         cfgmgr32hook_init(avs::game::DLL_INSTANCE);
     }

src/spice2x/games/sdvx/sdvx.cpp

@@ -21,6 +21,7 @@
 #include "util/socd_cleaner.h"
 #include "util/time.h"
 #include "util/libutils.h"
+#include "util/sysutils.h"
 #include "misc/eamuse.h"
 #include "misc/nativetouchhook.h"
 #include "misc/wintouchemu.h"
@@ -354,7 +355,7 @@ namespace games::sdvx {
         if (!GRAPHICS_WINDOWED && D3D9_ADAPTER.has_value() && is_valkyrie_model()) {
             SHOW_VM_MONITOR_WARNING = true;
             log_warning(
-                "iidx",
+                "sdvx",
                 "\n\n"
                 "!!! using -dxmainadapter option with Valkyrie mode is NOT          !!!\n"
                 "!!! recommended due to known compatibility issues with the game     !!!\n"
@@ -466,6 +467,7 @@ namespace games::sdvx {
                 ENABLE_COM_PORT_SCAN_HOOK = true;
             }
         }
+        
 #endif
 
         // ASIO device hook
@@ -501,6 +503,9 @@ namespace games::sdvx {
         // remove log spam
         logger::hook_add(sdvx64_spam_remover, nullptr);
 
+        if (is_valkyrie_model()) {
+            sysutils::hook_EnumDisplayDevicesA();
+        }
 #endif
     }
 

src/spice2x/launcher/launcher.cpp

@@ -473,6 +473,9 @@ int main_implementation(int argc, char *argv[]) {
     if (options[launcher::Options::spice2x_SDVXSubRedraw].value_bool()) {
         SUBSCREEN_FORCE_REDRAW = true;
     }
+    if (options[launcher::Options::SDVXSubMonitorOverride].is_active()) {
+        sysutils::SECOND_MONITOR_OVERRIDE = options[launcher::Options::SDVXSubMonitorOverride].value_text();
+    }
     if (options[launcher::Options::LoadIIDXModule].value_bool()) {
         attach_iidx = true;
     }
@@ -1127,6 +1130,9 @@ int main_implementation(int argc, char *argv[]) {
         options[launcher::Options::SDVXWindowedSubscreenAlwaysOnTop].value_bool()) {
         GRAPHICS_WSUB_ALWAYS_ON_TOP = true;
     }
+    if (options[launcher::Options::IIDXSubMonitorOverride].is_active()) {
+        sysutils::SECOND_MONITOR_OVERRIDE = options[launcher::Options::IIDXSubMonitorOverride].value_text();
+    }
 
     if (options[launcher::Options::spice2x_JubeatLegacyTouch].value_bool()) {
         games::jb::TOUCH_LEGACY_BOX = true;

src/spice2x/launcher/options.cpp

@@ -928,6 +928,17 @@ static const std::vector<OptionDefinition> OPTION_DEFINITIONS = {
             {"bottomright", "for landscape"},
         },
     },
+    {
+        // SDVXSubMonitorOverride
+        .title = "SDVX Subscreen Monitor Override",
+        .name = "sdvxsubmonitor",
+        .desc = "If you have three or more monitors, this option can be set to tell the game which monitor is the subscreen.",
+        .type = OptionType::Text,
+        .setting_name = "\\\\.\\DISPLAY3",
+        .game_name = "Sound Voltex",
+        .category = "Monitor",
+        .picker = OptionPickerType::Monitor,
+    },
     {
         .title = "Force Load DDR Module",
         .name = "ddr",
@@ -2343,6 +2354,17 @@ static const std::vector<OptionDefinition> OPTION_DEFINITIONS = {
         .game_name = "Beatmania IIDX",
         .category = "Game Options",
     },
+    {
+        // IIDXSubMonitorOverride
+        .title = "IIDX TDJ Subscreen Monitor Override",
+        .name = "iidxsubmonitor",
+        .desc = "If you have three or more monitors, this option can be set to tell the game which monitor is the subscreen.",
+        .type = OptionType::Text,
+        .setting_name = "\\\\.\\DISPLAY3",
+        .game_name = "Beatmania IIDX",
+        .category = "Monitor",
+        .picker = OptionPickerType::Monitor,
+    },
     {
         // spice2x_IIDXEmulateSubscreenKeypadTouch
         .title = "IIDX TDJ Subscreen Keypad Touch Emulation",

src/spice2x/launcher/options.h

@@ -92,6 +92,7 @@ namespace launcher {
             SDVXDigitalKnobSocd,
             spice2x_SDVXAsioDriver,
             spice2x_SDVXSubPos,
+            SDVXSubMonitorOverride,
             LoadDDRModule,
             DDR43Mode,
             DDRSkipCodecRegisteration,
@@ -243,6 +244,7 @@ namespace launcher {
             DRSRGBCameraHook,
             spice2x_IIDXNativeTouch,
             spice2x_IIDXNoSub,
+            IIDXSubMonitorOverride,
             spice2x_IIDXEmulateSubscreenKeypadTouch,
             spice2x_AutoCard,
             spice2x_LowLatencySharedAudio,

src/spice2x/util/sysutils.cpp

@@ -10,6 +10,9 @@
 #include <windows.h>
 #undef WIN32_NO_STATUS
 
+#include "hooks/graphics/graphics.h"
+#include "avs/game.h"
+#include "util/detour.h"
 #include "util/libutils.h"
 #include "util/logging.h"
 #include "util/utils.h"
@@ -86,6 +89,9 @@ namespace sysutils {
     );
     static GetSystemFirmwareTable_t GetSystemFirmwareTable = nullptr;
 
+    std::string SECOND_MONITOR_OVERRIDE = "";
+    static decltype(EnumDisplayDevicesA) *EnumDisplayDevicesA_orig = nullptr;
+
     void print_smbios() {
         DWORD bytes_written = 0;
         DWORD table_size = 0;
@@ -216,16 +222,13 @@ namespace sysutils {
         if (adapter->StateFlags & DISPLAY_DEVICE_MIRRORING_DRIVER) {
             return;
         }
-        if (!(adapter->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP)) {
-            return;
-        }
         std::string prefix("device");
         if (is_monitor) {
             prefix = "    adapter";
         }
         log_misc("gpuinfo", "{} {} device name   : {}", prefix.c_str(), index, adapter->DeviceName);
         log_misc("gpuinfo", "{} {} device string : {}", prefix.c_str(), index, adapter->DeviceString);
-        log_dbug("gpuinfo", "{} {} flags         : 0x{:x}", prefix.c_str(), index, adapter->StateFlags);
+        log_misc("gpuinfo", "{} {} flags         : 0x{:x}", prefix.c_str(), index, adapter->StateFlags);
 
         if (!is_monitor)  {
             // get extended info for better friendly name of monitors
@@ -264,6 +267,11 @@ namespace sysutils {
                 index,
                 (adapter->StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) ? "yes" : "no");
 
+            log_misc(
+                "gpuinfo", "{} {} is attached   : {}",
+                prefix.c_str(),
+                index,
+                (adapter->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) ? "yes" : "no");
         }
     }
 
@@ -409,4 +417,119 @@ namespace sysutils {
         static const std::vector<MonitorEntry> monitors = enumerate_monitors_internal();
         return monitors;
     }
+
+    static
+    BOOL
+    __stdcall
+    EnumDisplayDevicesA_hook(
+        LPCSTR lpDevice,
+        DWORD iDevNum,
+        PDISPLAY_DEVICEA lpDisplayDevice,
+        DWORD dwFlags
+    ) {
+        if (EnumDisplayDevicesA_orig == nullptr) {
+            return false;
+        }
+
+        // caller is enumerating monitors (adapters), not devices
+        if (lpDevice != nullptr) {
+            log_misc("sysutils", "EnumDisplayDevicesA: returning original results for device {} [{}]", lpDevice, iDevNum);
+            return EnumDisplayDevicesA_orig(lpDevice, iDevNum, lpDisplayDevice, dwFlags);
+        }
+
+        // call the original first
+        const auto result_orig = EnumDisplayDevicesA_orig(nullptr, iDevNum, lpDisplayDevice, dwFlags);
+        if (!result_orig) {
+            return result_orig;
+        }
+
+        log_misc(
+            "sysutils",
+            "EnumDisplayDevicesA_orig: {} [{}], StateFlags={:#x}",
+            lpDisplayDevice->DeviceName,
+            iDevNum,
+            lpDisplayDevice->StateFlags);
+
+        // this one is not relevant
+        if ((lpDisplayDevice->StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) == 0) {
+            return result_orig;
+        }
+
+        // if this is for the primary device, just return it
+        if (lpDisplayDevice->StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) {
+            log_misc(
+                "sysutils",
+                "EnumDisplayDevicesA: returning original results for primary monitor [{}], {}, result={}",
+                iDevNum,
+                lpDisplayDevice->DeviceName,
+                result_orig);
+            return result_orig;
+        }
+
+        // this is not 100% confirmed, but hope that this helps in cases where second monitors get used
+        // even though NumberOfAdaptersInGroup was set to 1 on hybrid laptops
+        if (GRAPHICS_FORCE_SINGLE_ADAPTER) {
+            lpDisplayDevice->StateFlags &= ~DISPLAY_DEVICE_ATTACHED_TO_DESKTOP;
+            log_info(
+                "sysutils",
+                "hiding this monitor, {} @ index {} (-graphics-single-adapter)",
+                lpDisplayDevice->DeviceName,
+                iDevNum);
+            return result_orig;
+        }
+
+        // for the second device (subscreen)...
+        if (SECOND_MONITOR_OVERRIDE.empty()) {
+            // if there is no user override, we find the device with lowest index (that isn't primary)
+            for (DWORD i = 0;; i++) {
+                DISPLAY_DEVICEA device = {};
+                device.cb = sizeof(device);
+                const auto result = EnumDisplayDevicesA_orig(nullptr, i, &device, 0);
+                if (!result) {
+                    break;
+                }
+                if (device.StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE) {
+                    // primary, skip this
+                    continue;
+                }
+                if ((device.StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP) == 0) {
+                    // disconnected, skip this as well
+                    continue;
+                }
+                if (std::string(lpDisplayDevice->DeviceName) == device.DeviceName) {
+                    log_info(
+                        "sysutils",
+                        "returning second monitor {} @ index {}",
+                        lpDisplayDevice->DeviceName,
+                        iDevNum);
+                    return result_orig;
+                } else {
+                    // otherwise, fall through and hide this monitor
+                    break;
+                }
+            }
+        } else if (SECOND_MONITOR_OVERRIDE == lpDisplayDevice->DeviceName) {
+            // if there is one preferred by the user, use this one, and hide others
+            log_info(
+                "sysutils",
+                "returning second monitor, {} @ index {} (-sysutilssubmonitor)",
+                lpDisplayDevice->DeviceName,
+                iDevNum);
+            return result_orig;
+        }
+
+        // this device should not be used; pretend that it's not connected
+        log_info(
+            "sysutils",
+            "hiding this monitor, {} @ index {}",
+            lpDisplayDevice->DeviceName,
+            iDevNum);
+        lpDisplayDevice->StateFlags &= ~DISPLAY_DEVICE_ATTACHED_TO_DESKTOP;
+        return result_orig;
+    }
+
+    void hook_EnumDisplayDevicesA() {
+        EnumDisplayDevicesA_orig = detour::iat_try(
+             "EnumDisplayDevicesA", EnumDisplayDevicesA_hook, avs::game::DLL_INSTANCE);
+    }
 }

src/spice2x/util/sysutils.h

@@ -15,4 +15,7 @@ namespace sysutils {
     };
 
     const std::vector<MonitorEntry> &enumerate_monitors();
+
+    extern std::string SECOND_MONITOR_OVERRIDE;
+    void hook_EnumDisplayDevicesA();    
 }