iidx: camera hook improvements (#841)

## Link to GitHub Issue or related Pull Request, if one exists
#201

## Description of change

- Adds MJPEG camera support using Media Foundation decoding.
- Moves camera capture to asynchronous Source Reader callbacks.
- Supports native NV12 and YUY2 capture, preferring target-sized native
formats over MJPEG when possible.
- Improves capture and rendering performance through callback-paced
reads, bulk frame uploads, optimized row-based flips, and lazy
allocation of flip textures.
- Makes runtime media-type changes interrupt pending reads and safely
flush and reconfigure the Source Reader.
- Improves automatic mode selection based on aspect ratio, proximity to
1280x720, frame rate, and capture format.
- Hardens camera shutdown, flush handling, media-type changes, and
repeated capture failures.

## Testing
Tested with two cameras in tdj

src/spice2x/games/iidx/camera.cpp

@@ -534,7 +534,7 @@ namespace games::iidx {
                         if (mediaTypePointer && mediaTypePointer->IsString()) {
                             std::string mediaType = mediaTypePointer->GetString();
                             if (mediaType.length() > 0) {
-                                camera->m_selectedMediaTypeDescription = mediaType;
+                                camera->SetSelectedMediaTypeDescription(mediaType);
                                 camera->m_useAutoMediaType = false;
                             } else {
                                 camera->m_useAutoMediaType = true;
@@ -547,7 +547,7 @@ namespace games::iidx {
                             std::string drawModeString = drawModePointer->GetString();
                             for (int j = 0; j < DRAW_MODE_SIZE; j++) {
                                 if (DRAW_MODE_LABELS[j].compare(drawModeString) == 0) {
-                                    camera->m_drawMode = (LocalCameraDrawMode) j;
+                                    camera->m_drawMode.store((LocalCameraDrawMode) j);
                                     break;
                                 }
                             }
@@ -556,12 +556,12 @@ namespace games::iidx {
                         // Flip
                         auto flipHorizontalPointer = rapidjson::Pointer(root + "/" + symLink + "/FlipHorizontal").Get(doc);
                         if (flipHorizontalPointer && flipHorizontalPointer->IsBool()) {
-                            camera->m_flipHorizontal = flipHorizontalPointer->GetBool();
+                            camera->m_flipHorizontal.store(flipHorizontalPointer->GetBool());
                         }
 
                         auto flipVerticalPointer = rapidjson::Pointer(root + "/" + symLink + "/FlipVertical").Get(doc);
                         if (flipVerticalPointer && flipVerticalPointer->IsBool()) {
-                            camera->m_flipVertical = flipVerticalPointer->GetBool();
+                            camera->m_flipVertical.store(flipVerticalPointer->GetBool());
                         }
 
                         // Allow manual control
@@ -625,15 +625,17 @@ namespace games::iidx {
             if (camera->m_useAutoMediaType) {
                 rapidjson::Pointer(root + "MediaType").Set(doc, "");
             } else {
-                rapidjson::Pointer(root + "MediaType").Set(doc, camera->m_selectedMediaTypeDescription);
+                const auto mediaTypeDescription = camera->GetSelectedMediaTypeDescription();
+                rapidjson::Pointer(root + "MediaType").Set(doc, mediaTypeDescription);
             }
 
             // Draw Mode
-            rapidjson::Pointer(root + "DrawMode").Set(doc, DRAW_MODE_LABELS[camera->m_drawMode]);
+            const auto drawMode = camera->m_drawMode.load();
+            rapidjson::Pointer(root + "DrawMode").Set(doc, DRAW_MODE_LABELS[drawMode]);
 
             // Flip
-            rapidjson::Pointer(root + "FlipHorizontal").Set(doc, camera->m_flipHorizontal);
-            rapidjson::Pointer(root + "FlipVertical").Set(doc, camera->m_flipVertical);
+            rapidjson::Pointer(root + "FlipHorizontal").Set(doc, camera->m_flipHorizontal.load());
+            rapidjson::Pointer(root + "FlipVertical").Set(doc, camera->m_flipVertical.load());
 
             // Manual control
             rapidjson::Pointer(root + "AllowManualControl").Set(doc, camera->m_allowManualControl);

src/spice2x/games/iidx/local_camera.h

@@ -2,6 +2,7 @@
 
 #if SPICE64 && !SPICE_XP
 
+#include <atomic>
 #include <d3d9.h>
 #include <dxva2api.h>
 #include <mutex>
@@ -66,6 +67,8 @@ extern std::string CAMERA_CONTROL_LABELS[];
 extern std::string DRAW_MODE_LABELS[];
 
 namespace games::iidx {
+    class IIDXCameraSourceReaderCallback;
+
     namespace Camera {
         struct PlayVideoCamera {
             IDirect3DTexture9** d3d9_texture(const uintptr_t offset) {
@@ -87,7 +90,7 @@ namespace games::iidx {
 
     class IIDXLocalCamera {
     protected:
-        virtual ~IIDXLocalCamera() {};
+        virtual ~IIDXLocalCamera();
 
         LONG m_nRefCount;
         CRITICAL_SECTION m_critsec;
@@ -101,9 +104,14 @@ namespace games::iidx {
         // For reading frames from Camera
         IMFMediaSource *m_pSource = nullptr;
         IMFSourceReader *m_pSourceReader = nullptr;
+        IMFSourceReaderEx *m_pSourceReaderEx = nullptr;
+        IIDXCameraSourceReaderCallback *m_pSourceReaderCallback = nullptr;
+        std::mutex m_mediaTypeMutex;
+        IMFMediaType *m_pendingMediaType = nullptr;
+        int m_selectedMediaTypeIndex = 0;
+        std::string m_selectedMediaTypeDescription = "";
 
         // Camera Format information
-        double                  m_frameRate = 0;
         LONG                    m_cameraWidth;
         LONG                    m_cameraHeight;
 
@@ -129,6 +137,12 @@ namespace games::iidx {
         LPDIRECT3DTEXTURE9 m_conversionTexture = nullptr;
         IDirect3DSurface9 *m_pConversionSurf = nullptr;
 
+        // upload surface for decoded camera frames returned in system memory
+        IDirect3DSurface9 *m_pDecodedSurf = nullptr;
+        GUID m_decodedSubtype = GUID_NULL;
+        GUID m_outputSubtype = GUID_NULL;
+        bool m_drawErrorLogged = false;
+
         // Texture for custom transform (e.g. horizontal flip)
         LPDIRECT3DTEXTURE9 m_transformTexture = nullptr;
         IDirect3DSurface9 *m_pTransformSurf = nullptr;
@@ -153,21 +167,19 @@ namespace games::iidx {
         BOOL m_initialized = false;
 
         // True if all the setup steps succeeded
-        BOOL m_active = false;
+        std::atomic_bool m_active = false;
 
         // Media type select
         std::vector<MediaTypeInfo> m_mediaTypeInfos = {};
-        int m_selectedMediaTypeIndex = 0;
         bool m_useAutoMediaType = true;
         IMFMediaType *m_pAutoMediaType = nullptr;
-        std::string m_selectedMediaTypeDescription = "";
         bool m_allowManualControl = false;
 
-        LocalCameraDrawMode m_drawMode = DrawModeCrop4_3;
+        std::atomic<LocalCameraDrawMode> m_drawMode = DrawModeCrop4_3;
 
         // Render processing
-        bool m_flipHorizontal = false;
-        bool m_flipVertical = false;
+        std::atomic_bool m_flipHorizontal = false;
+        std::atomic_bool m_flipVertical = false;
 
         IIDXLocalCamera(
             std::string name,
@@ -184,11 +196,14 @@ namespace games::iidx {
         HRESULT GetCameraControlProp(int index, CameraControlProp *pProp);
         HRESULT SetCameraControlProp(int index, long value, long flags);
         HRESULT ResetCameraControlProps();
-        HRESULT FlushDrawCommands();
         std::string GetName();
         std::string GetFriendlyName();
         std::string GetSymLink();
+        int GetSelectedMediaTypeIndex();
+        std::string GetSelectedMediaTypeDescription();
+        void SetSelectedMediaTypeDescription(const std::string &description);
         HRESULT ChangeMediaType(IMFMediaType *pType);
+        void RequestMediaType(IMFMediaType *pType);
         HRESULT StartCapture();
         void UpdateDrawRect();
 
@@ -197,12 +212,22 @@ namespace games::iidx {
         void CreateThread();
         MediaTypeInfo GetMediaTypeInfo(IMFMediaType *pType);
         std::string GetVideoFormatName(GUID subtype);
-        HRESULT TryMediaType(IMFMediaType *pType, UINT32 *pBestWidth, double *pBestFrameRate);
+        static bool CompareMediaTypes(const MediaTypeInfo &a, const MediaTypeInfo &b);
+        bool MatchesPreferredAspect(const MediaTypeInfo &info) const;
+        static bool IsBetterAutoType(const MediaTypeInfo &candidate, const MediaTypeInfo &current);
+        IMFMediaType *FindBestNativeAutoType(bool requirePreferredAspect) const;
+        IMFMediaType *FindBestAutoType(const GUID &subtype, bool requirePreferredAspect) const;
+        HRESULT ValidateMediaType(IMFMediaType *pType);
         HRESULT InitTargetTexture();
+        HRESULT EnsureTransformTextures();
         HRESULT InitCameraControl();
+        void SetSelectedMediaType(int index, const std::string &description);
+        bool HasPendingMediaType();
+        HRESULT ApplyPendingMediaType();
+        HRESULT UploadDecodedSample(IMFMediaBuffer *pSrcBuffer);
         HRESULT DrawSample(IMFMediaBuffer *pSrcBuffer);
         HRESULT ReadSample();
-        LPDIRECT3DTEXTURE9 Render();
+        HRESULT Render();
     };
 }
 

src/spice2x/games/iidx/mf_wrappers.cpp

@@ -15,6 +15,10 @@ namespace games::iidx {
         _In_ UINT32 cInitialSize
         );
 
+    typedef HRESULT (__stdcall * MFCreateMediaType_t)(
+        _Out_ IMFMediaType** ppMFType
+        );
+
     typedef HRESULT (__stdcall * MFEnumDeviceSources_t)(
         _In_ IMFAttributes* pAttributes,
         _Outptr_result_buffer_(*pcSourceActivate) IMFActivate*** pppSourceActivate,
@@ -35,6 +39,7 @@ namespace games::iidx {
         );
 
     static MFCreateAttributes_t MFCreateAttributes = nullptr;
+    static MFCreateMediaType_t MFCreateMediaType = nullptr;
     static MFEnumDeviceSources_t MFEnumDeviceSources = nullptr;
     static MFCreateSourceReaderFromMediaSource_t MFCreateSourceReaderFromMediaSource = nullptr;
     static MFGetService_t MFGetService = nullptr;
@@ -66,6 +71,12 @@ namespace games::iidx {
             log_fatal("mf_wrappers", "MFCreateAttributes failed to hook");
         }
 
+        MFCreateMediaType = (MFCreateMediaType_t)
+            libutils::get_proc(mfplat_dll, "MFCreateMediaType");
+        if (!MFCreateMediaType) {
+            log_fatal("mf_wrappers", "MFCreateMediaType failed to hook");
+        }
+
         MFEnumDeviceSources = (MFEnumDeviceSources_t)
             libutils::get_proc(mf_dll, "MFEnumDeviceSources");
         if (!MFEnumDeviceSources) {
@@ -92,6 +103,11 @@ namespace games::iidx {
         return MFCreateAttributes(ppMFAttributes, cInitialSize);
     }
 
+    HRESULT WrappedMFCreateMediaType (
+        _Out_ IMFMediaType** ppMFType) {
+        return MFCreateMediaType(ppMFType);
+    }
+
     HRESULT WrappedMFEnumDeviceSources (
         _In_ IMFAttributes* pAttributes,
         _Outptr_result_buffer_(*pcSourceActivate) IMFActivate*** pppSourceActivate,

src/spice2x/games/iidx/mf_wrappers.h

@@ -12,6 +12,9 @@ namespace games::iidx {
         _Out_ IMFAttributes** ppMFAttributes,
         _In_ UINT32 cInitialSize);
 
+    HRESULT WrappedMFCreateMediaType (
+        _Out_ IMFMediaType** ppMFType);
+
     HRESULT WrappedMFEnumDeviceSources (
         _In_ IMFAttributes* pAttributes,
         _Outptr_result_buffer_(*pcSourceActivate) IMFActivate*** pppSourceActivate,

src/spice2x/overlay/windows/camera_control.cpp

@@ -82,9 +82,14 @@ namespace overlay::windows {
         ImGui::Text("Rendering");
 
         // Media Type Selector
-        int selectedMediaTypeIndex = selectedCamera->m_selectedMediaTypeIndex;
+        int selectedMediaTypeIndex = selectedCamera->GetSelectedMediaTypeIndex();
         auto numMediaTypes = selectedCamera->m_mediaTypeInfos.size();
 
+        if (numMediaTypes == 0) {
+            ImGui::TextColored(ImVec4(1.f, 1.f, 0.f, 1.f), "%s", "No supported media types");
+            return;
+        }
+
         auto selectedMediaTypeIndexChanged = ImGui::BeginCombo(
             "Media Type", selectedCamera->m_mediaTypeInfos.at(selectedMediaTypeIndex).description.c_str()
         );
@@ -103,9 +108,9 @@ namespace overlay::windows {
             ImGui::EndCombo();
         }
 
-        if (selectedMediaTypeIndexChanged && selectedMediaTypeIndex != selectedCamera->m_selectedMediaTypeIndex) {
+        if (selectedMediaTypeIndexChanged && selectedMediaTypeIndex != selectedCamera->GetSelectedMediaTypeIndex()) {
             selectedCamera->m_useAutoMediaType = false;
-            selectedCamera->ChangeMediaType(selectedCamera->m_mediaTypeInfos.at(selectedMediaTypeIndex).p_mediaType);
+            selectedCamera->RequestMediaType(selectedCamera->m_mediaTypeInfos.at(selectedMediaTypeIndex).p_mediaType);
         }
 
         // Auto media type
@@ -114,13 +119,14 @@ namespace overlay::windows {
         if (ImGui::Checkbox("Auto##MediaType", &isAutoMediaType)) {
             selectedCamera->m_useAutoMediaType = isAutoMediaType;
             if (isAutoMediaType) {
-                selectedCamera->ChangeMediaType(selectedCamera->m_pAutoMediaType);
+                selectedCamera->RequestMediaType(selectedCamera->m_pAutoMediaType);
             }
         }
 
         // Draw mode
-        int selectedDrawModeIndex = selectedCamera->m_drawMode;
-        if (ImGui::BeginCombo("Draw Mode", DRAW_MODE_LABELS[selectedCamera->m_drawMode].c_str())) {
+        const auto drawMode = selectedCamera->m_drawMode.load();
+        int selectedDrawModeIndex = drawMode;
+        if (ImGui::BeginCombo("Draw Mode", DRAW_MODE_LABELS[drawMode].c_str())) {
             for (size_t i = 0; i < DRAW_MODE_SIZE; i++) {
                 const bool is_selected = (selectedDrawModeIndex == (int) i);
                 if (ImGui::Selectable(DRAW_MODE_LABELS[i].c_str(), is_selected)) {
@@ -142,15 +148,25 @@ namespace overlay::windows {
             "Letterbox to 4:3: Like Letterbox, but target 4:3"
             );
 
-        if (selectedDrawModeIndex != selectedCamera->m_drawMode) {
-            selectedCamera->m_drawMode = (LocalCameraDrawMode)selectedDrawModeIndex;
+        if (selectedDrawModeIndex != selectedCamera->m_drawMode.load()) {
+            selectedCamera->m_drawMode.store((LocalCameraDrawMode)selectedDrawModeIndex);
             selectedCamera->UpdateDrawRect();
         }
 
         ImGui::AlignTextToFramePadding();
-        ImGui::Checkbox("Horizontal Flip", &selectedCamera->m_flipHorizontal);
+        bool flipHorizontal = selectedCamera->m_flipHorizontal.load();
+        if (ImGui::Checkbox("Horizontal Flip", &flipHorizontal)) {
+            selectedCamera->m_flipHorizontal.store(flipHorizontal);
+        }
         ImGui::SameLine();
-        ImGui::Checkbox("Vertical Flip", &selectedCamera->m_flipVertical);
+        bool flipVertical = selectedCamera->m_flipVertical.load();
+        if (ImGui::Checkbox("Vertical Flip", &flipVertical)) {
+            selectedCamera->m_flipVertical.store(flipVertical);
+        }
+        ImGui::SameLine();
+        ImGui::HelpMarker(
+            "Flips are done by the CPU and can be very expensive, potentially causing frame drops! "
+            "Prefer to use camera's built-in flip if available.");
 
         // Camera control parameters
         ImGui::Separator();