sdvx: option to disable Live2D (#777)

## Link to GitHub Issue or related Pull Request, if one exists
n/a

## Description of change
Adds an option to disable Live2D. Can be disabled everywhere, or just
during a song.

## Testing
Tested EG final and recent Nabla.

src/spice2x/hooks/graphics/backends/d3d9/d3d9_device.cpp

@@ -1,7 +1,11 @@
 #include "d3d9_device.h"
 
+#include <algorithm>
 #include <cassert>
+#include <climits>
 #include <mutex>
+#include <unordered_map>
+#include <vector>
 
 #include "avs/game.h"
 #include "games/gitadora/gitadora.h"
@@ -12,6 +16,7 @@
 #include "cfg/screen_resize.h"
 
 #include "d3d9_backend.h"
+#include "d3d9_live2d.h"
 #include "d3d9_texture.h"
 
 #ifndef SPICE64
@@ -1301,6 +1306,9 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::DrawPrimitive(
         UINT PrimitiveCount)
 {
     WRAP_DEBUG;
+    if (d3d9_live2d::should_skip_draw()) [[unlikely]] {
+        return D3D_OK;
+    }
     CHECK_RESULT(pReal->DrawPrimitive(PrimitiveType, StartVertex, PrimitiveCount));
 }
 
@@ -1313,6 +1321,9 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::DrawIndexedPrimitive(
         UINT PrimitiveCount)
 {
     WRAP_DEBUG;
+    if (d3d9_live2d::should_skip_draw()) [[unlikely]] {
+        return D3D_OK;
+    }
     CHECK_RESULT(pReal->DrawIndexedPrimitive(
             PrimitiveType, BaseVertexIndex,
             MinVertexIndex, NumVertices,
@@ -1328,6 +1339,9 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::DrawPrimitiveUP(
     WRAP_DEBUG_FMT("DrawPrimitiveUP({}, {}, {}, {})",
             PrimitiveType, PrimitiveCount,
             fmt::ptr(pVertexStreamZeroData), VertexStreamZeroStride);
+    if (d3d9_live2d::should_skip_draw()) [[unlikely]] {
+        return D3D_OK;
+    }
     CHECK_RESULT(pReal->DrawPrimitiveUP(
             PrimitiveType, PrimitiveCount,
             pVertexStreamZeroData, VertexStreamZeroStride));
@@ -1344,6 +1358,9 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::DrawIndexedPrimitiveUP(
         UINT VertexStreamZeroStride)
 {
     WRAP_DEBUG;
+    if (d3d9_live2d::should_skip_draw()) [[unlikely]] {
+        return D3D_OK;
+    }
     CHECK_RESULT(pReal->DrawIndexedPrimitiveUP(
             PrimitiveType, MinVertexIndex, NumVertices, PrimitiveCount, pIndexData,
             IndexDataFormat, pVertexStreamZeroData, VertexStreamZeroStride));
@@ -1403,7 +1420,14 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::CreateVertexShader(
         IDirect3DVertexShader9 **ppShader)
 {
     WRAP_VERBOSE_FMT("CreateVertexShader({})", fmt::ptr(pFunction));
-    CHECK_RESULT(pReal->CreateVertexShader(pFunction, ppShader));
+    HRESULT ret = pReal->CreateVertexShader(pFunction, ppShader);
+    if (SUCCEEDED(ret) && ppShader != nullptr) {
+        d3d9_live2d::on_create_vertex_shader(*ppShader, pFunction);
+    }
+    if (GRAPHICS_LOG_HRESULT && FAILED(ret)) [[unlikely]] {
+        log_warning("graphics::d3d9", "{} failed, hr={}", __FUNCTION__, FMT_HRESULT(ret));
+    }
+    return ret;
 }
 
 HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::SetVertexShader(
@@ -1411,6 +1435,8 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::SetVertexShader(
 {
     WRAP_DEBUG_FMT("SetVertexShader({})", fmt::ptr(pShader));
 
+    d3d9_live2d::on_set_vertex_shader(pShader);
+
 #ifndef SPICE64
 
     // diagonal line fix
@@ -1557,13 +1583,21 @@ HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::CreatePixelShader(
         IDirect3DPixelShader9 **ppShader)
 {
     WRAP_VERBOSE_FMT("CreatePixelShader({})", fmt::ptr(pFunction));
-    CHECK_RESULT(pReal->CreatePixelShader(pFunction, ppShader));
+    HRESULT ret = pReal->CreatePixelShader(pFunction, ppShader);
+    if (SUCCEEDED(ret) && ppShader != nullptr) {
+        d3d9_live2d::on_create_pixel_shader(*ppShader, pFunction);
+    }
+    if (GRAPHICS_LOG_HRESULT && FAILED(ret)) [[unlikely]] {
+        log_warning("graphics::d3d9", "{} failed, hr={}", __FUNCTION__, FMT_HRESULT(ret));
+    }
+    return ret;
 }
 
 HRESULT STDMETHODCALLTYPE WrappedIDirect3DDevice9::SetPixelShader(
         IDirect3DPixelShader9 *pShader)
 {
     WRAP_DEBUG_FMT("SetPixelShader({})", fmt::ptr(pShader));
+    d3d9_live2d::on_set_pixel_shader(pShader);
     CHECK_RESULT(pReal->SetPixelShader(pShader));
 }
 

src/spice2x/hooks/graphics/backends/d3d9/d3d9_live2d.cpp

@@ -0,0 +1,144 @@
+#include "d3d9_live2d.h"
+
+// only the Live2D-capable SDVX versions are 64-bit, so the entire implementation
+// is compiled out of 32-bit builds (the header supplies inline no-op stubs there).
+#ifdef SPICE64
+
+#include <cstdint>
+#include <unordered_set>
+
+#include "hooks/graphics/graphics.h"
+
+// how the Live2D draw filtering works
+// ------------------------------------
+// SDVX draws its Live2D characters with a small, fixed set of pixel and
+// vertex shaders. to skip those draws (and save GPU) we have to recognise them at
+// the exact moment the game issues a draw call. the d3d9 device hooks feed three
+// kinds of events into this module:
+//
+//   1. shader creation  (on_create_pixel_shader / on_create_vertex_shader)
+//        the game compiles its shaders once at load. we can't trust the shader
+//        *object pointer* to identify a shader (it's just a heap address that
+//        varies per run and can be recycled), so instead we hash the shader's
+//        D3D9 *bytecode* - that fingerprint is stable across runs because the
+//        game ships the same shaders. if the hash matches a known Live2D shader
+//        we remember that object pointer in g_live2d_shaders.
+//
+//   2. shader binding   (on_set_pixel_shader / on_set_vertex_shader)
+//        whenever the game binds a shader we look it up in that set once and cache
+//        the yes/no answer in g_cur_ps_is_live2d / g_cur_vs_is_live2d. binds happen
+//        far less often than draws, so this is where the lookup cost lives.
+//
+//   3. draw call        (should_skip_draw, called from every Draw* hook)
+//        the per-draw question "is this a Live2D draw?" is then just reading those
+//        two cached bools - no hashing, no map lookups. if the skip is currently
+//        active (see graphics_sdvx_live2d_should_skip) and either bound shader is
+//        Live2D, the Draw* hook drops the call instead of forwarding it.
+//
+// everything is gated on the feature being enabled (mode != Off); when it's Off
+// every entry point is a single predicted-not-taken branch. d3d9 rendering for a
+// device is single-threaded, so none of this state needs locking.
+
+namespace {
+
+// shader state is tracked whenever the feature might act (mode != Off) so the
+// known-shader set is populated before a song starts. when Off, every entry
+// point is a single cheap branch.
+bool tracking_enabled() {
+    return GRAPHICS_SDVX_LIVE2D_MODE != SdvxLive2dMode::Off;
+}
+
+// the set of shader objects (pixel or vertex) whose bytecode matched a known
+// Live2D fingerprint. only matching shaders are stored, so this stays tiny.
+std::unordered_set<void *> g_live2d_shaders;
+
+// whether the currently-bound shaders are known Live2D shaders. cached at set
+// time so the per-draw check is just two bool reads.
+bool g_cur_ps_is_live2d = false;
+bool g_cur_vs_is_live2d = false;
+
+// FNV-1a 64 over a D3D9 shader token stream (ends with D3DSIO_END = 0x0000FFFF)
+uint64_t bytecode_hash(const DWORD *func) {
+    if (func == nullptr) {
+        return 0;
+    }
+    const DWORD *p = func;
+    const DWORD *cap = func + 65536; // safety bound
+    while (p < cap && *p != 0x0000FFFF) {
+        p++;
+    }
+    const size_t n_bytes = ((size_t)(p - func) + 1) * sizeof(DWORD);
+    uint64_t h = 1469598103934665603ULL;
+    const auto *bytes = reinterpret_cast<const uint8_t *>(func);
+    for (size_t i = 0; i < n_bytes; i++) {
+        h ^= bytes[i];
+        h *= 1099511628211ULL;
+    }
+    return h;
+}
+
+// known SDVX Live2D shader bytecode hashes (4 pixel + 3 vertex). stable
+// across runs because the game ships fixed shaders. the two sets are disjoint so
+// a single shader can be classified by its own hash alone.
+bool hash_is_live2d(uint64_t hash) {
+    switch (hash) {
+    case 0x75c89951817421a4ULL: // pixel: dominant model draw (~4.9M prims/120f in-song)
+    case 0x2d7ce428c6b4775dULL: // pixel: masked model draw
+    case 0x3ce00cc6111c10e7ULL: // pixel: mask generation
+    case 0x8bb3a2f37150ac34ULL: // pixel: mask generation (variant)
+    case 0xe9cf898c331e2a51ULL: // vertex
+    case 0x94dc84e7b7c0f437ULL: // vertex
+    case 0xc872937c5cc04309ULL: // vertex
+        return true;
+    }
+    return false;
+}
+
+// classify a shader at creation time and record it if it is Live2D. erasing on a
+// miss keeps the set correct if the runtime reuses a freed shader pointer.
+void classify_shader(void *shader, const DWORD *func) {
+    if (hash_is_live2d(bytecode_hash(func))) {
+        g_live2d_shaders.insert(shader);
+    } else {
+        g_live2d_shaders.erase(shader);
+    }
+}
+
+} // namespace
+
+namespace d3d9_live2d {
+
+// stage 1: fingerprint each shader as the game creates it
+void on_create_vertex_shader(IDirect3DVertexShader9 *shader, const DWORD *func) {
+    if (tracking_enabled() && shader != nullptr) [[unlikely]] {
+        classify_shader(shader, func);
+    }
+}
+
+void on_create_pixel_shader(IDirect3DPixelShader9 *shader, const DWORD *func) {
+    if (tracking_enabled() && shader != nullptr) [[unlikely]] {
+        classify_shader(shader, func);
+    }
+}
+
+// stage 2: remember whether the just-bound shader is a Live2D one
+void on_set_vertex_shader(IDirect3DVertexShader9 *shader) {
+    if (tracking_enabled()) [[unlikely]] {
+        g_cur_vs_is_live2d = g_live2d_shaders.count(shader) != 0;
+    }
+}
+
+void on_set_pixel_shader(IDirect3DPixelShader9 *shader) {
+    if (tracking_enabled()) [[unlikely]] {
+        g_cur_ps_is_live2d = g_live2d_shaders.count(shader) != 0;
+    }
+}
+
+// stage 3: drop the draw if the skip is active and a Live2D shader is bound
+bool should_skip_draw() {
+    return graphics_sdvx_live2d_should_skip() && (g_cur_ps_is_live2d || g_cur_vs_is_live2d);
+}
+
+} // namespace d3d9_live2d
+
+#endif // SPICE64

src/spice2x/hooks/graphics/backends/d3d9/d3d9_live2d.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <windows.h>
+#include <d3d9.h>
+
+// SDVX Live2D draw-skip support for the D3D9 backend.
+//
+// SDVX renders its Live2D navigator / in-song character through a fixed set of
+// shaders. when the skip is active (see graphics_sdvx_live2d_should_skip)
+// the matching draw calls are dropped to save GPU. shaders are identified by a
+// stable hash of their D3D9 bytecode (object pointers vary per run, the bytecode
+// does not). the hashes were captured with the draw-call fingerprinting tool.
+//
+// every entry point is a no-op unless the feature is enabled (mode != Off), and
+// d3d9 rendering for a device is single-threaded, so none of this needs locking.
+namespace d3d9_live2d {
+
+#ifdef SPICE64
+
+    // record a shader's bytecode fingerprint at creation time
+    void on_create_vertex_shader(IDirect3DVertexShader9 *shader, const DWORD *func);
+    void on_create_pixel_shader(IDirect3DPixelShader9 *shader, const DWORD *func);
+
+    // remember the currently-bound shaders
+    void on_set_vertex_shader(IDirect3DVertexShader9 *shader);
+    void on_set_pixel_shader(IDirect3DPixelShader9 *shader);
+
+    // true if the current draw call should be dropped (skip active AND the bound
+    // shaders identify it as SDVX Live2D)
+    bool should_skip_draw();
+
+#else // !SPICE64
+
+    // only the Live2D-capable SDVX versions are 64-bit; on 32-bit every entry point
+    // compiles away to nothing, so the d3d9 device hooks need no #ifdefs at their
+    // call sites.
+    inline void on_create_vertex_shader(IDirect3DVertexShader9 *, const DWORD *) {}
+    inline void on_create_pixel_shader(IDirect3DPixelShader9 *, const DWORD *) {}
+    inline void on_set_vertex_shader(IDirect3DVertexShader9 *) {}
+    inline void on_set_pixel_shader(IDirect3DPixelShader9 *) {}
+    inline bool should_skip_draw() { return false; }
+
+#endif // SPICE64
+}

src/spice2x/hooks/graphics/graphics.cpp

@@ -86,6 +86,10 @@ static bool monitor_layout_needs_reset = false;
 bool GRAPHICS_CAPTURE_CURSOR = false;
 bool GRAPHICS_LOG_HRESULT = false;
 bool GRAPHICS_SDVX_FORCE_720 = false;
+#ifdef SPICE64
+SdvxLive2dMode GRAPHICS_SDVX_LIVE2D_MODE = SdvxLive2dMode::Off;
+std::atomic<bool> GRAPHICS_SDVX_LIVE2D_IN_GAMEPLAY = false;
+#endif // SPICE64
 bool GRAPHICS_SHOW_CURSOR = false;
 bool GRAPHICS_WINDOWED = false;
 std::vector<HWND> GRAPHICS_WINDOWS;

src/spice2x/hooks/graphics/graphics.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <atomic>
 #include <string>
 #include <vector>
 #include <optional>
@@ -27,6 +28,33 @@ enum graphics_dx9on12_state {
     DX9ON12_FORCE_ON,
 };
 
+// SDVX Live2D suppression policy. the mode comes from the launcher
+// option; the in-gameplay flag is maintained by the SDVX scene-detection hook in
+// games/sdvx (which does not require the SDVX game module to be enabled). the
+// d3d9 backend reads graphics_sdvx_live2d_should_skip() on every draw.
+// only the Live2D-capable SDVX versions are 64-bit, so the whole feature is
+// compiled out of 32-bit builds.
+#ifdef SPICE64
+enum class SdvxLive2dMode {
+    Off,    // leave Live2D untouched (default)
+    Always, // always skip Live2D draws (also removes the menu navigator)
+    InGame, // skip Live2D draws only during a song
+};
+extern SdvxLive2dMode GRAPHICS_SDVX_LIVE2D_MODE;
+extern std::atomic<bool> GRAPHICS_SDVX_LIVE2D_IN_GAMEPLAY;
+
+inline bool graphics_sdvx_live2d_should_skip() {
+    switch (GRAPHICS_SDVX_LIVE2D_MODE) {
+        case SdvxLive2dMode::Always:
+            return true;
+        case SdvxLive2dMode::InGame:
+            return GRAPHICS_SDVX_LIVE2D_IN_GAMEPLAY.load(std::memory_order_relaxed);
+        default:
+            return false;
+    }
+}
+#endif // SPICE64
+
 // flag settings
 extern bool GRAPHICS_CAPTURE_CURSOR;
 extern bool GRAPHICS_LOG_HRESULT;