quartz: Fix deadlock with transform filter.

dlls/quartz/acmwrapper.c

@@ -65,11 +65,26 @@ static HRESULT ACMWrapper_ProcessSampleData(TransformFilterImpl* pTransformFilte
     MMRESULT res;
     HRESULT hr;
     LONGLONG tStart = -1, tStop = -1, tMed;
+    InputPin *pin = (InputPin *)pTransformFilter->ppPins[0];
+
+    EnterCriticalSection(&pTransformFilter->csFilter);
+    if (pTransformFilter->state == State_Stopped)
+    {
+        LeaveCriticalSection(&pTransformFilter->csFilter);
+        return VFW_E_WRONG_STATE;
+    }
+
+    if (pin->end_of_stream || pin->flushing)
+    {
+        LeaveCriticalSection(&pTransformFilter->csFilter);
+        return S_FALSE;
+    }
 
     hr = IMediaSample_GetPointer(pSample, &pbSrcStream);
     if (FAILED(hr))
     {
         ERR("Cannot get pointer to sample data (%x)\n", hr);
+        LeaveCriticalSection(&pTransformFilter->csFilter);
         return hr;
     }
 
@@ -94,8 +109,10 @@ static HRESULT ACMWrapper_ProcessSampleData(TransformFilterImpl* pTransformFilte
     TRACE("Sample data ptr = %p, size = %ld\n", pbSrcStream, (long)cbSrcStream);
 
     hr = IPin_ConnectionMediaType(This->tf.ppPins[0], &amt);
-    if (FAILED(hr)) {
+    if (FAILED(hr))
+    {
         ERR("Unable to retrieve media type\n");
+        LeaveCriticalSection(&pTransformFilter->csFilter);
         return hr;
     }
 
@@ -105,8 +122,10 @@ static HRESULT ACMWrapper_ProcessSampleData(TransformFilterImpl* pTransformFilte
     while(hr == S_OK && ash.cbSrcLength)
     {
         hr = OutputPin_GetDeliveryBuffer((OutputPin*)This->tf.ppPins[1], &pOutSample, NULL, NULL, 0);
-	if (FAILED(hr)) {
+        if (FAILED(hr))
+        {
             ERR("Unable to get delivery buffer (%x)\n", hr);
+            LeaveCriticalSection(&pTransformFilter->csFilter);
             return hr;
         }
         IMediaSample_SetPreroll(pOutSample, preroll);
@@ -186,7 +205,9 @@ static HRESULT ACMWrapper_ProcessSampleData(TransformFilterImpl* pTransformFilte
         }
         TRACE("Sample stop time: %u.%03u\n", (DWORD)(tStart/10000000), (DWORD)((tStart/10000)%1000));
 
+        LeaveCriticalSection(&This->tf.csFilter);
         hr = OutputPin_SendSample((OutputPin*)This->tf.ppPins[1], pOutSample);
+        EnterCriticalSection(&This->tf.csFilter);
 
         if (hr != S_OK && hr != VFW_E_NOT_CONNECTED) {
             if (FAILED(hr))
@@ -210,6 +231,7 @@ error:
     This->lasttime_real = tStop;
     This->lasttime_sent = tMed;
 
+    LeaveCriticalSection(&pTransformFilter->csFilter);
     return hr;
 }
 

dlls/quartz/avidec.c

@@ -80,12 +80,26 @@ static HRESULT AVIDec_ProcessSampleData(TransformFilterImpl* pTransformFilter, I
     DWORD cbSrcStream;
     LPBYTE pbSrcStream;
     LONGLONG tStart, tStop;
+    InputPin *pin = (InputPin *)pTransformFilter->ppPins[0];
+
+    EnterCriticalSection(&This->tf.csFilter);
+    if (This->tf.state == State_Stopped)
+    {
+        LeaveCriticalSection(&This->tf.csFilter);
+        return VFW_E_WRONG_STATE;
+    }
+
+    if (pin->end_of_stream || pin->flushing)
+    {
+        LeaveCriticalSection(&This->tf.csFilter);
+        return S_FALSE;
+    }
 
     hr = IMediaSample_GetPointer(pSample, &pbSrcStream);
     if (FAILED(hr))
     {
         ERR("Cannot get pointer to sample data (%x)\n", hr);
-        return hr;
+        goto error;
     }
 
     cbSrcStream = IMediaSample_GetActualDataLength(pSample);
@@ -137,16 +151,18 @@ static HRESULT AVIDec_ProcessSampleData(TransformFilterImpl* pTransformFilter, I
     else
         IMediaSample_SetTime(pOutSample, NULL, NULL);
 
+    LeaveCriticalSection(&This->tf.csFilter);
     hr = OutputPin_SendSample((OutputPin*)This->tf.ppPins[1], pOutSample);
-    if (hr != S_OK && hr != VFW_E_NOT_CONNECTED) {
+    if (hr != S_OK && hr != VFW_E_NOT_CONNECTED)
         ERR("Error sending sample (%x)\n", hr);
-	goto error;
-    }
+    IMediaSample_Release(pOutSample);
+    return hr;
 
 error:
     if (pOutSample)
         IMediaSample_Release(pOutSample);
 
+    LeaveCriticalSection(&This->tf.csFilter);
     return hr;
 }
 

dlls/quartz/dsoundrender.c

@@ -245,8 +245,19 @@ static HRESULT DSoundRender_Sample(LPVOID iface, IMediaSample * pSample)
      * pause completion here, but for sound playing a single frame doesn't make sense
      */
 
+    EnterCriticalSection(&This->csFilter);
+
+    if (This->pInputPin->end_of_stream || This->pInputPin->flushing)
+    {
+        LeaveCriticalSection(&This->csFilter);
+        return S_FALSE;
+    }
+
     if (This->state == State_Stopped)
+    {
+        LeaveCriticalSection(&This->csFilter);
         return VFW_E_WRONG_STATE;
+    }
 
     SetEvent(This->state_change);
 
@@ -279,11 +290,15 @@ static HRESULT DSoundRender_Sample(LPVOID iface, IMediaSample * pSample)
         WaitForSingleObject(This->blocked, INFINITE);
         EnterCriticalSection(&This->csFilter);
         if (This->state == State_Stopped)
+        {
+            LeaveCriticalSection(&This->csFilter);
             return VFW_E_WRONG_STATE;
+        }
 
         if (This->state == State_Paused)
         {
             /* Assuming we return because of flushing */
+            LeaveCriticalSection(&This->csFilter);
             return S_OK;
         }
     }
@@ -304,7 +319,9 @@ static HRESULT DSoundRender_Sample(LPVOID iface, IMediaSample * pSample)
     }
 #endif
 
-    return DSoundRender_SendSampleData(This, pbSrcStream, cbSrcStream);
+    hr = DSoundRender_SendSampleData(This, pbSrcStream, cbSrcStream);
+    LeaveCriticalSection(&This->csFilter);
+    return hr;
 }
 
 static HRESULT DSoundRender_QueryAccept(LPVOID iface, const AM_MEDIA_TYPE * pmt)

dlls/quartz/nullrenderer.c

@@ -82,29 +82,17 @@ static const IMemInputPinVtbl MemInputPin_Vtbl =
 
 static HRESULT NullRenderer_Sample(LPVOID iface, IMediaSample * pSample)
 {
-    LPBYTE pbSrcStream = NULL;
-    long cbSrcStream = 0;
-    REFERENCE_TIME tStart, tStop;
-    HRESULT hr;
+    NullRendererImpl *This = (NullRendererImpl *)iface;
+    HRESULT hr = S_OK;
 
     TRACE("%p %p\n", iface, pSample);
 
-    hr = IMediaSample_GetPointer(pSample, &pbSrcStream);
-    if (FAILED(hr))
-    {
-        ERR("Cannot get pointer to sample data (%x)\n", hr);
-        return hr;
-    }
-
-    hr = IMediaSample_GetTime(pSample, &tStart, &tStop);
-    if (FAILED(hr))
-        ERR("Cannot get sample time (%x)\n", hr);
-
-    cbSrcStream = IMediaSample_GetActualDataLength(pSample);
-
-    TRACE("val %p %ld\n", pbSrcStream, cbSrcStream);
+    EnterCriticalSection(&This->csFilter);
+    if (This->pInputPin->flushing || This->pInputPin->end_of_stream)
+        hr = S_FALSE;
+    LeaveCriticalSection(&This->csFilter);
 
-    return S_OK;
+    return hr;
 }
 
 static HRESULT NullRenderer_QueryAccept(LPVOID iface, const AM_MEDIA_TYPE * pmt)

dlls/quartz/pin.c

@@ -803,13 +803,7 @@ HRESULT WINAPI MemInputPin_Receive(IMemInputPin * iface, IMediaSample * pSample)
 
     /* this trace commented out for performance reasons */
     /*TRACE("(%p/%p)->(%p)\n", This, iface, pSample);*/
-
-    EnterCriticalSection(This->pin.pCritSec);
-    if (!This->end_of_stream && !This->flushing)
     hr = This->fnSampleProc(This->pin.pUserData, pSample);
-    else
-        hr = S_FALSE;
-    LeaveCriticalSection(This->pin.pCritSec);
     return hr;
 }
 

dlls/quartz/transform.c

@@ -48,18 +48,6 @@ static const IPinVtbl TransformFilter_InputPin_Vtbl;
 static const IMemInputPinVtbl MemInputPin_Vtbl; 
 static const IPinVtbl TransformFilter_OutputPin_Vtbl;
 
-static HRESULT TransformFilter_Sample(LPVOID iface, IMediaSample * pSample)
-{
-    TransformFilterImpl *This = (TransformFilterImpl *)iface;
-
-    TRACE("%p %p\n", iface, pSample);
-
-    if (This->state == State_Stopped)
-        return S_FALSE;
-
-    return This->pFuncsTable->pfnProcessSampleData(This, pSample);
-}
-
 static HRESULT TransformFilter_Input_QueryAccept(LPVOID iface, const AM_MEDIA_TYPE * pmt)
 {
     TransformFilterImpl* This = (TransformFilterImpl*)iface;
@@ -187,7 +175,7 @@ HRESULT TransformFilter_Create(TransformFilterImpl* pTransformFilter, const CLSI
     piOutput.pFilter = (IBaseFilter *)pTransformFilter;
     lstrcpynW(piOutput.achName, wcsOutputPinName, sizeof(piOutput.achName) / sizeof(piOutput.achName[0]));
 
-    hr = InputPin_Construct(&TransformFilter_InputPin_Vtbl, &piInput, TransformFilter_Sample, pTransformFilter, TransformFilter_Input_QueryAccept, NULL, &pTransformFilter->csFilter, NULL, &pTransformFilter->ppPins[0]);
+    hr = InputPin_Construct(&TransformFilter_InputPin_Vtbl, &piInput, (SAMPLEPROC_PUSH)pFuncsTable->pfnProcessSampleData, pTransformFilter, TransformFilter_Input_QueryAccept, NULL, &pTransformFilter->csFilter, NULL, &pTransformFilter->ppPins[0]);
 
     if (SUCCEEDED(hr))
     {

dlls/quartz/videorenderer.c

@@ -362,9 +362,15 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
     long cbSrcStream = 0;
     REFERENCE_TIME tStart, tStop;
     HRESULT hr;
+    EnterCriticalSection(&This->csFilter);
+    if (This->pInputPin->flushing || This->pInputPin->end_of_stream)
+        hr = S_FALSE;
 
     if (This->state == State_Stopped)
+    {
+        LeaveCriticalSection(&This->csFilter);
         return VFW_E_WRONG_STATE;
+    }
 
     TRACE("%p %p\n", iface, pSample);
 
@@ -386,6 +392,7 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
     if (IMediaSample_IsPreroll(pSample) == S_OK)
     {
         This->rtLastStop = tStop;
+        LeaveCriticalSection(&This->csFilter);
         return S_OK;
     }
 
@@ -393,6 +400,7 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
     if (FAILED(hr))
     {
         ERR("Cannot get pointer to sample data (%x)\n", hr);
+        LeaveCriticalSection(&This->csFilter);
         return hr;
     }
 
@@ -422,11 +430,17 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
         EnterCriticalSection(&This->csFilter);
         This->sample_held = NULL;
         if (This->state == State_Paused)
+        {
             /* Flushing */
+            LeaveCriticalSection(&This->csFilter);
             return S_OK;
+        }
         if (This->state == State_Stopped)
+        {
+            LeaveCriticalSection(&This->csFilter);
             return VFW_E_WRONG_STATE;
         }
+    }
 
     if (This->pClock && This->state == State_Running)
     {
@@ -456,6 +470,7 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
                   (DWORD)(time / 10000000), (DWORD)((time / 10000)%1000),
                   (DWORD)(trefstop / 10000000), (DWORD)((trefstop / 10000)%1000) );
             This->rtLastStop = tStop;
+            LeaveCriticalSection(&This->csFilter);
             return S_OK;
         }
     }
@@ -463,6 +478,7 @@ static HRESULT VideoRenderer_Sample(LPVOID iface, IMediaSample * pSample)
 
     VideoRenderer_SendSampleData(This, pbSrcStream, cbSrcStream);
 
+    LeaveCriticalSection(&This->csFilter);
     return S_OK;
 }