wasapi: address buffer overflow when using -wasapishared (#758)

## Link to GitHub Issue or related Pull Request, if one exists
Building on #745 

## Description of change
With `-wasapishared`, some users see `AUDCLNT_E_BUFFER_TOO_LARGE`
(`0x88890006`).

The existing one-device-period buffer clamp wasn't enough on endpoints
with a small shared buffer; the shared buffer is double-buffered, so a
full-buffer write often exceeds the free space.

### Fix

Added a FIFO bridge to `SharedRedirect` that decouples the game's
per-event writes from the shared engine's clock:

- **`GetBuffer`** hands the game a pointer into the FIFO tail to write
in place.
- **`ReleaseBuffer`** commits the write and drains `min(pending,
device_free)` frames to the device - so a write can never exceed what
the device accepts, structurally preventing the overflow on any
endpoint.
- **`GetCurrentPadding`** reports the FIFO fill level (capped to the
reported buffer size) so poll/timer-driven games pace correctly against
the virtual buffer.

## Testing

src/spice2x/hooks/audio/backends/wasapi/audio_client.cpp

@@ -295,6 +295,13 @@ HRESULT STDMETHODCALLTYPE WrappedIAudioClient::Initialize(
     copy_wave_format(&hooks::audio::FORMAT, device_format);
     copy_wave_format(&this->device_format, device_format);
 
+    // arm the shared-mode buffer bridge so the redirected game's full-buffer writes are paced to
+    // the device instead of overflowing the shared buffer (AUDCLNT_E_BUFFER_TOO_LARGE).
+    if (this->shared.redirected_from_exclusive) {
+        this->shared.enable_bridge(
+                device_format->nChannels * (device_format->wBitsPerSample / 8));
+    }
+
     return ret;
 }
 HRESULT STDMETHODCALLTYPE WrappedIAudioClient::GetBufferSize(UINT32 *pNumBufferFrames) {
@@ -379,6 +386,12 @@ HRESULT STDMETHODCALLTYPE WrappedIAudioClient::GetCurrentPadding(UINT32 *pNumPad
         *pNumPaddingFrames = this->resample.padding_device_to_game(*pNumPaddingFrames);
     }
 
+    // shared-mode bridge: the game writes into a FIFO, not the device buffer, so report the FIFO's
+    // fill level rather than the device's padding (which is in a different buffer space).
+    if (SUCCEEDED(ret) && this->shared.bridge_enabled() && pNumPaddingFrames) {
+        *pNumPaddingFrames = this->shared.virtual_padding();
+    }
+
     CHECK_RESULT(ret);
 }
 HRESULT STDMETHODCALLTYPE WrappedIAudioClient::IsFormatSupported(

src/spice2x/hooks/audio/backends/wasapi/audio_render_client.cpp

@@ -12,60 +12,6 @@
 
 const char CLASS_NAME[] = "WrappedIAudioRenderClient";
 
-// scale every sample of an interleaved device buffer by `gain`, clamped to the format's range.
-// supports 16/24/32-bit PCM and 32-bit float; other formats are left untouched.
-static void apply_gain(BYTE *buffer, UINT32 frames, const WAVEFORMATEXTENSIBLE &fmt, float gain) {
-    const WAVEFORMATEX &f = fmt.Format;
-    const size_t samples = (size_t) frames * f.nChannels;
-
-    bool is_float = is_ieee_float(&f);
-
-    if (is_float && f.wBitsPerSample == 32) {
-        auto p = reinterpret_cast<float *>(buffer);
-        for (size_t i = 0; i < samples; i++) {
-            p[i] = std::clamp(p[i] * gain, -1.0f, 1.0f);
-        }
-        return;
-    }
-
-    switch (f.wBitsPerSample) {
-        case 16: {
-            auto p = reinterpret_cast<int16_t *>(buffer);
-            for (size_t i = 0; i < samples; i++) {
-                p[i] = (int16_t) std::clamp((int) std::lround(p[i] * gain), -32768, 32767);
-            }
-            break;
-        }
-        case 24: {
-            // packed 24-bit little-endian
-            for (size_t i = 0; i < samples; i++) {
-                BYTE *s = buffer + i * 3;
-                int32_t v = s[0] | (s[1] << 8) | (s[2] << 16);
-                if (v & 0x800000) {
-                    v |= ~0xFFFFFF; // sign extend
-                }
-                int64_t scaled = std::clamp<int64_t>(
-                    std::llround((double) v * gain), -8388608, 8388607);
-                s[0] = scaled & 0xFF;
-                s[1] = (scaled >> 8) & 0xFF;
-                s[2] = (scaled >> 16) & 0xFF;
-            }
-            break;
-        }
-        case 32: {
-            auto p = reinterpret_cast<int32_t *>(buffer);
-            for (size_t i = 0; i < samples; i++) {
-                p[i] = (int32_t) std::clamp(
-                    std::llround((double) p[i] * gain),
-                    (long long) INT32_MIN, (long long) INT32_MAX);
-            }
-            break;
-        }
-        default:
-            break;
-    }
-}
-
 HRESULT STDMETHODCALLTYPE WrappedIAudioRenderClient::QueryInterface(REFIID riid, void **ppvObj) {
     if (ppvObj == nullptr) {
         return E_POINTER;
@@ -129,6 +75,14 @@ HRESULT STDMETHODCALLTYPE WrappedIAudioRenderClient::GetBuffer(UINT32 NumFramesR
     // device buffer is acquired in ReleaseBuffer once the converted frame count is known.
     } else if (this->client->resample.enabled) {
         CHECK_RESULT(this->client->resample.get_buffer(NumFramesRequested, ppData));
+
+    // shared-mode redirect bridge: point the game at the FIFO tail it can always fill, decoupling
+    // its per-event writes from the shared engine's clock. the real device buffer is acquired in
+    // ReleaseBuffer and filled only as fast as the device frees space (see SharedRedirect::drain).
+    } else if (this->client->shared.bridge_enabled()) {
+        *ppData = this->client->shared.begin_write(NumFramesRequested);
+
+        return S_OK;
     }
 
     // call original
@@ -183,6 +137,19 @@ HRESULT STDMETHODCALLTYPE WrappedIAudioRenderClient::ReleaseBuffer(UINT32 NumFra
                 hooks::audio::VOLUME_BOOST);
     }
 
+    // shared-mode redirect bridge: queue the game's write and drain it to the device at the
+    // device's own pace, so a full-buffer write never overflows the shared buffer.
+    if (this->client->shared.bridge_enabled()) {
+        this->client->shared.commit_write(
+                NumFramesWritten, (dwFlags & AUDCLNT_BUFFERFLAGS_SILENT) != 0);
+
+        return this->client->shared.drain(
+                pReal,
+                this->client->pReal,
+                this->client->device_format,
+                hooks::audio::VOLUME_BOOST);
+    }
+
     // resolve the real device buffer for whichever path produced the audio
     BYTE *device_buffer;
     if (this->client->downmix.enabled) {

src/spice2x/hooks/audio/backends/wasapi/shared.cpp

@@ -1,10 +1,13 @@
 #include "shared.h"
 
+#include <algorithm>
+
 #include <audioclient.h>
 
 #include "hooks/audio/audio.h"
 #include "util/logging.h"
 
+#include "util.h"
 #include "defs.h"
 
 namespace hooks::audio {
@@ -58,8 +61,9 @@ namespace hooks::audio {
     }
 
     UINT32 SharedRedirect::clamp_buffer_size(IAudioClient *real, uint32_t sample_rate,
-            UINT32 device_frames) const {
+            UINT32 device_frames) {
         if (!this->redirected_from_exclusive || real == nullptr || sample_rate == 0 || device_frames == 0) {
+            this->reported_frames = device_frames;
             return device_frames;
         }
 
@@ -69,10 +73,115 @@ namespace hooks::audio {
         if (SUCCEEDED(real->GetDevicePeriod(&period, nullptr)) && period > 0) {
             const UINT32 period_frames = (UINT32) ((period * sample_rate) / 10000000);
             if (period_frames > 0 && period_frames < device_frames) {
+                this->reported_frames = period_frames;
                 return period_frames;
             }
         }
 
+        this->reported_frames = device_frames;
         return device_frames;
     }
+
+    void SharedRedirect::enable_bridge(int frame_bytes) {
+        if (!this->redirected_from_exclusive || frame_bytes <= 0) {
+            return;
+        }
+
+        this->frame_bytes = frame_bytes;
+        this->device_buffer_frames = 0;
+        this->fifo.clear();
+
+        log_info("audio::wasapi", "shared-mode buffer bridge enabled (frame size {} bytes)",
+                frame_bytes);
+    }
+
+    BYTE *SharedRedirect::begin_write(UINT32 frames) {
+        // reserve space at the FIFO tail and let the game write straight into it - no scratch copy.
+        this->pending_write_offset = this->fifo.size();
+        this->fifo.resize(this->pending_write_offset + (size_t) frames * this->frame_bytes);
+
+        return this->fifo.data() + this->pending_write_offset;
+    }
+
+    void SharedRedirect::commit_write(UINT32 frames, bool silent) {
+        // trim the tail reservation to the frames actually written; zero it in place if silent.
+        const size_t end = this->pending_write_offset + (size_t) frames * this->frame_bytes;
+        if (silent) {
+            std::fill(this->fifo.begin() + this->pending_write_offset,
+                    this->fifo.begin() + end, (BYTE) 0);
+        }
+        this->fifo.resize(end);
+    }
+
+    UINT32 SharedRedirect::pending_frames() const {
+        if (this->frame_bytes <= 0) {
+            return 0;
+        }
+
+        return (UINT32) (this->fifo.size() / this->frame_bytes);
+    }
+
+    UINT32 SharedRedirect::virtual_padding() const {
+        const UINT32 pending = this->pending_frames();
+        return this->reported_frames > 0 ? std::min(pending, this->reported_frames) : pending;
+    }
+
+    HRESULT SharedRedirect::drain(IAudioRenderClient *real, IAudioClient *client,
+            const WAVEFORMATEXTENSIBLE &device_format, float boost) {
+        if (!this->bridge_enabled()) {
+            return S_OK;
+        }
+
+        // cache the real device buffer size once; it is fixed for the life of the stream.
+        if (this->device_buffer_frames == 0) {
+            if (FAILED(client->GetBufferSize(&this->device_buffer_frames))
+                    || this->device_buffer_frames == 0) {
+                return S_OK;
+            }
+        }
+
+        const UINT32 pending = this->pending_frames();
+        if (pending == 0) {
+            return S_OK;
+        }
+
+        // push only as many frames as the device currently has free, keeping the rest queued. this
+        // self-paces to the engine's real consumption so a full-buffer write never overflows.
+        UINT32 padding = 0;
+        if (FAILED(client->GetCurrentPadding(&padding))) {
+            return S_OK;
+        }
+        const UINT32 device_free = this->device_buffer_frames > padding
+                ? this->device_buffer_frames - padding
+                : 0;
+        if (device_free == 0) {
+            return S_OK;
+        }
+
+        const UINT32 to_write = std::min(pending, device_free);
+
+        BYTE *dev = nullptr;
+        HRESULT ret = real->GetBuffer(to_write, &dev);
+        if (FAILED(ret) || dev == nullptr) {
+            return ret;
+        }
+
+        const size_t bytes = (size_t) to_write * this->frame_bytes;
+        std::copy(this->fifo.begin(), this->fifo.begin() + bytes, dev);
+
+        // mute the first few buffers to avoid a startup pop, then apply the volume boost.
+        if (this->buffers_to_mute > 0) {
+            std::fill(dev, dev + bytes, (BYTE) 0);
+            this->buffers_to_mute--;
+        } else if (boost != 1.0f) {
+            apply_gain(dev, to_write, device_format, boost);
+        }
+
+        ret = real->ReleaseBuffer(to_write, 0);
+
+        // drop the frames just handed to the device from the front of the FIFO.
+        this->fifo.erase(this->fifo.begin(), this->fifo.begin() + bytes);
+
+        return ret;
+    }
 }

src/spice2x/hooks/audio/backends/wasapi/shared.h

@@ -1,15 +1,19 @@
 #pragma once
 
 #include <cstdint>
+#include <vector>
 
 #include <windows.h>
+#include <mmreg.h>
 #include <audioclient.h>
 
+struct IAudioRenderClient;
+
 namespace hooks::audio {
 
-    // The -wasapishared option redirects an exclusive WASAPI stream to shared mode. lets other apps
+    // The -wasapishared option redirects an exclusive WASAPI stream to shared mode, so other apps
-    // play sound and works on devices that can't open the exclusive format, at the cost of some
+    // can play sound and devices that can't open the exclusive format still work, at the cost of
-    // latency. Only PCM / float is converted; bitstream (AC-3 / DTS) is left alone.
+    // some latency. Only PCM / float is converted; bitstream (AC-3 / DTS) is left alone.
     struct SharedRedirect {
 
         // true once apply() has redirected an exclusive request. gates the buffer clamp; stays false
@@ -23,9 +27,57 @@ namespace hooks::audio {
         // redirect an exclusive request to shared mode. caller must have checked wants() first.
         void apply(AUDCLNT_SHAREMODE *share_mode, DWORD *stream_flags, REFERENCE_TIME *periodicity);
 
-        // clamp a reported buffer size to one device period. some games write the whole buffer per
+        // clamp a reported buffer size to one device period. the FIFO bridge below is what prevents
-        // event, which overflows shared-mode buffering (AUDCLNT_E_BUFFER_TOO_LARGE -> stutter); one
+        // the overflow; this just keeps the game's per-event writes small so the bridge adds minimal
-        // period always fits. a no-op unless an exclusive request was redirected.
+        // latency. caches the chosen value for virtual_padding. a no-op unless redirected.
-        UINT32 clamp_buffer_size(IAudioClient *real, uint32_t sample_rate, UINT32 device_frames) const;
+        UINT32 clamp_buffer_size(IAudioClient *real, uint32_t sample_rate, UINT32 device_frames);
+
+        // FIFO bridge: the redirected game writes a whole reported buffer per event paced by its own
+        // callback, not the shared engine clock, so a full-buffer write can intermittently exceed the
+        // double-buffered shared free space (AUDCLNT_E_BUFFER_TOO_LARGE). The game instead writes
+        // directly into a FIFO that is drained to the device only as fast as it frees space - the
+        // same free-space-clamped approach the timer-driven resampler uses.
+
+        // arm the bridge once the redirected stream is initialized. frame_bytes is one frame's size
+        // in the game's (== device, via AUTOCONVERTPCM) format.
+        void enable_bridge(int frame_bytes);
+
+        // whether the FIFO bridge is active (a redirect was applied and armed).
+        bool bridge_enabled() const { return this->frame_bytes > 0; }
+
+        // reserve `frames` at the FIFO tail and hand the game a pointer into it to write in place.
+        // must be paired with commit_write, which trims the reservation to the frames written.
+        BYTE *begin_write(UINT32 frames);
+
+        // trim the reservation from begin_write to the `frames` actually written (zeroing if silent).
+        void commit_write(UINT32 frames, bool silent);
+
+        // padding to report to a game that polls GetCurrentPadding while the bridge is active: the
+        // FIFO fill level, capped to the reported buffer size so the game's free-space calculation
+        // (reported_buffer - padding) reflects room in the virtual buffer rather than the device's.
+        UINT32 virtual_padding() const;
+
+        // push as many queued frames as the real device has free, applying `boost`, keeping the rest
+        // for the next call. `real` is the wrapped render client's underlying interface; `client` is
+        // the underlying audio client used to query the device's free space.
+        HRESULT drain(IAudioRenderClient *real, IAudioClient *client,
+                const WAVEFORMATEXTENSIBLE &device_format, float boost);
+
+    private:
+
+        // frames currently queued in the FIFO and not yet handed to the device.
+        UINT32 pending_frames() const;
+
+        // FIFO bridge state (see enable_bridge). fifo holds audio queued for the device in the
+        // game's interleaved frame format; the game writes new frames directly into its tail between
+        // begin_write and commit_write. frame_bytes > 0 doubles as the "bridge armed" flag (see
+        // bridge_enabled). pending_write_offset marks the tail reservation handed to begin_write.
+        int frame_bytes = 0;
+        UINT32 device_buffer_frames = 0;
+        UINT32 reported_frames = 0;
+        int buffers_to_mute = 4;
+        size_t pending_write_offset = 0;
+        std::vector<BYTE> fifo;
     };
 }
+

src/spice2x/hooks/audio/backends/wasapi/util.cpp

@@ -8,6 +8,58 @@
 
 #include "defs.h"
 
+void apply_gain(BYTE *buffer, UINT32 frames, const WAVEFORMATEXTENSIBLE &fmt, float gain) {
+    const WAVEFORMATEX &f = fmt.Format;
+    const size_t samples = (size_t) frames * f.nChannels;
+
+    bool is_float = is_ieee_float(&f);
+
+    if (is_float && f.wBitsPerSample == 32) {
+        auto p = reinterpret_cast<float *>(buffer);
+        for (size_t i = 0; i < samples; i++) {
+            p[i] = std::clamp(p[i] * gain, -1.0f, 1.0f);
+        }
+        return;
+    }
+
+    switch (f.wBitsPerSample) {
+        case 16: {
+            auto p = reinterpret_cast<int16_t *>(buffer);
+            for (size_t i = 0; i < samples; i++) {
+                p[i] = (int16_t) std::clamp((int) std::lround(p[i] * gain), -32768, 32767);
+            }
+            break;
+        }
+        case 24: {
+            // packed 24-bit little-endian
+            for (size_t i = 0; i < samples; i++) {
+                BYTE *s = buffer + i * 3;
+                int32_t v = s[0] | (s[1] << 8) | (s[2] << 16);
+                if (v & 0x800000) {
+                    v |= ~0xFFFFFF; // sign extend
+                }
+                int64_t scaled = std::clamp<int64_t>(
+                    std::llround((double) v * gain), -8388608, 8388607);
+                s[0] = scaled & 0xFF;
+                s[1] = (scaled >> 8) & 0xFF;
+                s[2] = (scaled >> 16) & 0xFF;
+            }
+            break;
+        }
+        case 32: {
+            auto p = reinterpret_cast<int32_t *>(buffer);
+            for (size_t i = 0; i < samples; i++) {
+                p[i] = (int32_t) std::clamp(
+                    std::llround((double) p[i] * gain),
+                    (long long) INT32_MIN, (long long) INT32_MAX);
+            }
+            break;
+        }
+        default:
+            break;
+    }
+}
+
 std::string stream_flags_str(DWORD flags) {
     FLAGS_START(flags);
     FLAG(flags, AUDCLNT_STREAMFLAGS_CROSSPROCESS);

src/spice2x/hooks/audio/backends/wasapi/util.h

@@ -21,6 +21,10 @@ void print_format(AUDCLNT_SHAREMODE share_mode, DWORD stream_flags, REFERENCE_TI
 // IAudioClient::IsFormatSupported).
 void print_format(AUDCLNT_SHAREMODE share_mode, const WAVEFORMATEX *device_format);
 
+// scale every sample of an interleaved device buffer by `gain`, clamped to the format's range.
+// supports 16/24/32-bit PCM and 32-bit float; other formats are left untouched.
+void apply_gain(BYTE *buffer, UINT32 frames, const WAVEFORMATEXTENSIBLE &fmt, float gain);
+
 // detect IEEE float samples: WAVE_FORMAT_IEEE_FLOAT, or WAVE_FORMAT_EXTENSIBLE whose SubFormat is
 // KSDATAFORMAT_SUBTYPE_IEEE_FLOAT (Data1 == 3; _PCM has Data1 == 1)
 inline bool is_ieee_float(const WAVEFORMATEX *fmt) {