graphics: remove static import of d3d9 (#780)

## Link to GitHub Issue or related Pull Request, if one exists
Fixes #779
Regressed by #720 

## Description of change
#720 introduced a static dependency on DX9 which caused `d3d9.dll` to be
loaded at boot from `system32`. Some third party hooks (like
ifs_layeredfs and dxvk) rely on supplying a custom `d3d9.dll` in the DLL
search path (usually in modules) but this change caused Windows to skip
that check.

Remove the hard dependency on d3d9 and add a CMake check to ensure that
compiled binaries do not accidentally introduce new static imports in
the future.

## Testing
Confirmed that dxvk runs again.

src/spice2x/CMakeLists.txt

@@ -678,6 +678,33 @@ set(SOURCE_FILES ${SOURCE_FILES}
 
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} PREFIX "Source Files" FILES ${SOURCE_FILES})
 
+# guard against statically importing DLLs that must always be loaded dynamically:
+#   * DLLs users override via the modules directory (e.g. DXVK's d3d9.dll) - a
+#     static import loads the system copy at startup and preempts the override
+#     (issue #779).
+#   * Media Foundation DLLs (mf/mfplat/mfreadwrite) - a static import breaks
+#     Unity games.
+# the check runs objdump on each produced binary and fails the build if any
+# forbidden DLL is imported.
+set(SPICE_FORBIDDEN_STATIC_IMPORTS
+        d3d8.dll d3d9.dll d3d10core.dll d3d11.dll dxgi.dll opengl32.dll
+        mf.dll mfplat.dll mfreadwrite.dll)
+
+function(spice_guard_dll_imports target)
+    if(MSVC)
+        # objdump-based parsing assumes a GNU/LLVM toolchain
+        return()
+    endif()
+    add_custom_command(TARGET ${target} POST_BUILD
+            COMMAND ${CMAKE_COMMAND}
+                -DOBJDUMP=${CMAKE_OBJDUMP}
+                -DTARGET_FILE=$<TARGET_FILE:${target}>
+                "-DFORBIDDEN=${SPICE_FORBIDDEN_STATIC_IMPORTS}"
+                -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/check_no_static_dll_imports.cmake
+            VERBATIM
+            COMMENT "Checking ${target} for forbidden static DLL imports")
+endfunction()
+
 # spice.exe / spice_laa.exe shared objects
 ###########################################
 # spice.exe and spice_laa.exe are compiled identically; the only difference is
@@ -962,3 +989,16 @@ set_target_properties(spicetools_spice64 spicetools_spice64_linux spicetools_stu
         ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/archive64"
         LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/spicetools/64"
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/spicetools/64")
+
+# forbidden static DLL import guard
+###################################
+# apply the check to every executable and DLL produced by this project.
+get_property(spice_all_targets DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY BUILDSYSTEM_TARGETS)
+foreach(spice_target IN LISTS spice_all_targets)
+    get_target_property(spice_target_type ${spice_target} TYPE)
+    if(spice_target_type STREQUAL "EXECUTABLE"
+            OR spice_target_type STREQUAL "SHARED_LIBRARY"
+            OR spice_target_type STREQUAL "MODULE_LIBRARY")
+        spice_guard_dll_imports(${spice_target})
+    endif()
+endforeach()

src/spice2x/cfg/configurator.cpp

@@ -3,6 +3,7 @@
 #include <d3d9.h>
 
 #include "overlay/overlay.h"
+#include "util/libutils.h"
 #include "util/logging.h"
 
 namespace cfg {
@@ -35,7 +36,26 @@ namespace cfg {
             return false;
         }
 
-        IDirect3D9 *d3d = Direct3DCreate9(D3D_SDK_VERSION);
+        // load d3d9.dll dynamically rather than linking against it statically.
+        // a static import would force the system d3d9.dll to load at process
+        // startup for the main spice executable too (this file is shared with
+        // spice.exe), which loads system32\d3d9.dll before the modules directory
+        // is added to the DLL search path - preventing a user-supplied DXVK
+        // d3d9.dll in modules from ever loading for the game.
+        typedef IDirect3D9 *(WINAPI *Direct3DCreate9_t)(UINT);
+        HMODULE d3d9_module = libutils::try_library("d3d9.dll");
+        if (d3d9_module == nullptr) {
+            log_warning("configurator", "could not load d3d9.dll, falling back to software renderer");
+            return false;
+        }
+        auto Direct3DCreate9_fn =
+                reinterpret_cast<Direct3DCreate9_t>(libutils::try_proc(d3d9_module, "Direct3DCreate9"));
+        if (Direct3DCreate9_fn == nullptr) {
+            log_warning("configurator", "could not find Direct3DCreate9, falling back to software renderer");
+            return false;
+        }
+
+        IDirect3D9 *d3d = Direct3DCreate9_fn(D3D_SDK_VERSION);
         if (d3d == nullptr) {
             log_warning("configurator", "Direct3DCreate9 returned NULL, falling back to software renderer");
             return false;

src/spice2x/cmake/check_no_static_dll_imports.cmake

@@ -0,0 +1,72 @@
+# fails the build if a PE binary statically imports a forbidden DLL.
+#
+# some DLLs must never end up in spice's static import table, for two reasons:
+#
+#  1. user-overridable DLLs (e.g. DXVK's d3d9.dll): users drop their own copy
+#     into the modules directory to replace the system one. a static import
+#     forces the loader to load the SYSTEM copy at process startup - before the
+#     modules directory is added to the DLL search path and before the game DLL
+#     loads - so the user-supplied override never takes effect (see issue #779).
+#
+#  2. DLLs that break games when present (e.g. Media Foundation: mf/mfplat/
+#     mfreadwrite): a static import loads them eagerly and breaks Unity games.
+#
+# in both cases the DLL must instead be loaded dynamically (libutils::try_library
+# / GetProcAddress / delay load) so it is only pulled in when actually needed.
+#
+# invoked via `cmake -P` from a POST_BUILD step. required -D variables:
+#   OBJDUMP     - path to objdump (CMAKE_OBJDUMP)
+#   TARGET_FILE - path to the PE binary to inspect
+#   FORBIDDEN   - semicolon-separated list of lowercase DLL names to reject
+
+if(NOT OBJDUMP OR NOT EXISTS "${OBJDUMP}")
+    message(WARNING
+        "check_no_static_dll_imports: objdump not found, skipping import check for ${TARGET_FILE}")
+    return()
+endif()
+
+execute_process(
+    COMMAND "${OBJDUMP}" -p "${TARGET_FILE}"
+    OUTPUT_VARIABLE dump_output
+    RESULT_VARIABLE dump_result
+    ERROR_VARIABLE dump_error)
+
+if(NOT dump_result EQUAL 0)
+    message(WARNING
+        "check_no_static_dll_imports: objdump failed for ${TARGET_FILE}: ${dump_error}")
+    return()
+endif()
+
+# both GNU objdump and llvm-objdump print one "DLL Name: <name>" line per
+# statically imported DLL in their PE private-header dump.
+string(REGEX MATCHALL "DLL Name:[ \t]*[^\n\r]+" dll_lines "${dump_output}")
+
+set(violations "")
+foreach(line IN LISTS dll_lines)
+    string(REGEX REPLACE "DLL Name:[ \t]*" "" dll_name "${line}")
+    string(STRIP "${dll_name}" dll_name)
+    string(TOLOWER "${dll_name}" dll_name_lower)
+    if(dll_name_lower IN_LIST FORBIDDEN)
+        list(APPEND violations "${dll_name}")
+    endif()
+endforeach()
+
+if(violations)
+    list(REMOVE_DUPLICATES violations)
+    string(REPLACE ";" ", " violations_str "${violations}")
+    message(FATAL_ERROR
+        "static DLL import check FAILED for ${TARGET_FILE}\n"
+        "  forbidden static imports found: ${violations_str}\n"
+        "\n"
+        "  these DLLs must never be statically imported by spice:\n"
+        "    * user-overridable DLLs (e.g. DXVK d3d9.dll) - a static import loads the\n"
+        "      system copy at startup and preempts the modules override (issue #779).\n"
+        "    * Media Foundation DLLs (mf/mfplat/mfreadwrite) - a static import breaks\n"
+        "      Unity games.\n"
+        "\n"
+        "  fix: load the DLL dynamically instead - replace the direct API call with a\n"
+        "  libutils::try_library() + libutils::try_proc() lookup (or a delay load), then\n"
+        "  call through the resolved function pointer.")
+endif()
+
+message(STATUS "static DLL import check passed for ${TARGET_FILE}")