wined3d: Fail texture creation when invalid multisample settings are specified.

dlls/d3d11/texture.c

@@ -482,6 +482,8 @@ static HRESULT d3d_texture2d_init(struct d3d_texture2d *texture, struct d3d_devi
         WARN("Failed to create wined3d texture, hr %#x.\n", hr);
         wined3d_private_store_cleanup(&texture->private_store);
         wined3d_mutex_unlock();
+        if (hr == WINED3DERR_NOTAVAILABLE)
+            hr = E_INVALIDARG;
         return hr;
     }
     texture->desc.MipLevels = levels;

dlls/d3d9/device.c

@@ -1137,6 +1137,8 @@ static HRESULT d3d9_device_create_surface(struct d3d9_device *device, UINT width
     {
         wined3d_mutex_unlock();
         WARN("Failed to create texture, hr %#x.\n", hr);
+        if (hr == WINED3DERR_NOTAVAILABLE)
+            hr = D3DERR_INVALIDCALL;
         return hr;
     }
 

dlls/wined3d/surface.c

@@ -2879,12 +2879,6 @@ static void surface_prepare_rb(struct wined3d_surface *surface, const struct win
                         break;
                 }
             }
-            if (i == sizeof(format->multisample_types) * 8)
-            {
-                WARN("Unsupported quality level %u requested for WINED3D_MULTISAMPLE_NON_MASKABLE.\n",
-                        surface->resource.multisample_quality);
-                i = 1;
-            }
             samples = i + 1;
         }
         else

dlls/wined3d/texture.c

@@ -1458,6 +1458,27 @@ HRESULT CDECL wined3d_texture_create(struct wined3d_device *device, const struct
         return WINED3DERR_INVALIDCALL;
     }
 
+    if (desc->multisample_type != WINED3D_MULTISAMPLE_NONE)
+    {
+        const struct wined3d_format *format = wined3d_get_format(&device->adapter->gl_info, desc->format);
+
+        if (desc->multisample_type == WINED3D_MULTISAMPLE_NON_MASKABLE
+                && desc->multisample_quality >= wined3d_popcount(format->multisample_types))
+        {
+            WARN("Unsupported quality level %u requested for WINED3D_MULTISAMPLE_NON_MASKABLE.\n",
+                    desc->multisample_quality);
+            return WINED3DERR_NOTAVAILABLE;
+        }
+        if (desc->multisample_type != WINED3D_MULTISAMPLE_NON_MASKABLE
+                && (!(format->multisample_types & 1u << (desc->multisample_type - 1))
+                || desc->multisample_quality))
+        {
+            WARN("Unsupported multisample type %u quality %u requested.\n", desc->multisample_type,
+                    desc->multisample_quality);
+            return WINED3DERR_NOTAVAILABLE;
+        }
+    }
+
     if (!(object = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(*object))))
         return E_OUTOFMEMORY;