wined3d: Fix pixel format masks.

dlls/wined3d/context.c

@@ -2398,7 +2398,7 @@ static void context_setup_target(struct wined3d_context *context, struct wined3d
         if (old->id != new->id)
         {
             /* Disable blending when the alpha mask has changed and when a format doesn't support blending. */
-            if ((old->alpha_mask && !new->alpha_mask) || (!old->alpha_mask && new->alpha_mask)
+            if ((old->alpha_size && !new->alpha_size) || (!old->alpha_size && new->alpha_size)
                     || !(new->flags & WINED3DFMT_FLAG_POSTPIXELSHADER_BLENDING))
                 context_invalidate_state(context, STATE_RENDER(WINED3D_RS_ALPHABLENDENABLE));
 

dlls/wined3d/state.c

@@ -379,9 +379,9 @@ static GLenum gl_blend_factor(enum wined3d_blend factor, const struct wined3d_fo
          * returns 1.0, so WINED3D_BLEND_DESTALPHA becomes GL_ONE, and
          * WINED3D_BLEND_INVDESTALPHA becomes GL_ZERO. */
         case WINED3D_BLEND_DESTALPHA:
-            return dst_format->alpha_mask ? GL_DST_ALPHA : GL_ONE;
+            return dst_format->alpha_size ? GL_DST_ALPHA : GL_ONE;
         case WINED3D_BLEND_INVDESTALPHA:
-            return dst_format->alpha_mask ? GL_ONE_MINUS_DST_ALPHA : GL_ZERO;
+            return dst_format->alpha_size ? GL_ONE_MINUS_DST_ALPHA : GL_ZERO;
         case WINED3D_BLEND_SRCALPHASAT:
             return GL_SRC_ALPHA_SATURATE;
         case WINED3D_BLEND_BLENDFACTOR:
@@ -3253,7 +3253,7 @@ void tex_alphaop(struct wined3d_context *context, const struct wined3d_state *st
         {
             struct wined3d_surface *surf = surface_from_resource(texture->sub_resources[0]);
 
-            if (surf->CKeyFlags & WINEDDSD_CKSRCBLT && !surf->resource.format->alpha_mask)
+            if (surf->CKeyFlags & WINEDDSD_CKSRCBLT && !surf->resource.format->alpha_size)
             {
                 /* Color keying needs to pass alpha values from the texture through to have the alpha test work
                  * properly. On the other hand applications can still use texture combiners apparently. This code

dlls/wined3d/surface.c

@@ -371,6 +371,14 @@ void draw_textured_quad(const struct wined3d_surface *src_surface, struct wined3
     }
 }
 
+/* Works correctly only for <= 4 bpp formats. */
+static void get_color_masks(const struct wined3d_format *format, DWORD *masks)
+{
+    masks[0] = ((1 << format->red_size) - 1) << format->red_offset;
+    masks[1] = ((1 << format->green_size) - 1) << format->green_offset;
+    masks[2] = ((1 << format->blue_size) - 1) << format->blue_offset;
+}
+
 static HRESULT surface_create_dib_section(struct wined3d_surface *surface)
 {
     const struct wined3d_format *format = surface->resource.format;
@@ -456,9 +464,7 @@ static HRESULT surface_create_dib_section(struct wined3d_surface *surface)
         case WINED3DFMT_B5G6R5_UNORM:
         case WINED3DFMT_R16G16B16A16_UNORM:
             b_info->bmiHeader.biCompression = BI_BITFIELDS;
-            masks[0] = format->red_mask;
-            masks[1] = format->green_mask;
-            masks[2] = format->blue_mask;
+            get_color_masks(format, masks);
             break;
 
         default:
@@ -6971,9 +6977,11 @@ do { \
                 }
                 else
                 {
-                    keymask = src_format->red_mask
-                            | src_format->green_mask
-                            | src_format->blue_mask;
+                    DWORD masks[3];
+                    get_color_masks(src_format, masks);
+                    keymask = masks[0]
+                            | masks[1]
+                            | masks[2];
                 }
                 flags &= ~(WINEDDBLT_KEYSRC | WINEDDBLT_KEYDEST | WINEDDBLT_KEYSRCOVERRIDE | WINEDDBLT_KEYDESTOVERRIDE);
             }

dlls/wined3d/wined3d_private.h

@@ -2796,10 +2796,14 @@ struct wined3d_format
 {
     enum wined3d_format_id id;
 
-    DWORD red_mask;
-    DWORD green_mask;
-    DWORD blue_mask;
-    DWORD alpha_mask;
+    DWORD red_size;
+    DWORD green_size;
+    DWORD blue_size;
+    DWORD alpha_size;
+    DWORD red_offset;
+    DWORD green_offset;
+    DWORD blue_offset;
+    DWORD alpha_offset;
     UINT byte_count;
     BYTE depth_size;
     BYTE stencil_size;