wined3d: Add a slop for color keying.

dlls/wined3d/arb_program_shader.c

@@ -5756,8 +5756,9 @@ const struct wined3d_shader_backend_ops arb_program_shader_backend =
 
 /* ARB_fragment_program fixed function pipeline replacement definitions */
 #define ARB_FFP_CONST_TFACTOR           0
-#define ARB_FFP_CONST_COLOR_KEY         ((ARB_FFP_CONST_TFACTOR) + 1)
-#define ARB_FFP_CONST_SPECULAR_ENABLE   ((ARB_FFP_CONST_COLOR_KEY) + 1)
+#define ARB_FFP_CONST_COLOR_KEY_LOW     ((ARB_FFP_CONST_TFACTOR) + 1)
+#define ARB_FFP_CONST_COLOR_KEY_HIGH    ((ARB_FFP_CONST_COLOR_KEY_LOW) + 1)
+#define ARB_FFP_CONST_SPECULAR_ENABLE   ((ARB_FFP_CONST_COLOR_KEY_HIGH) + 1)
 #define ARB_FFP_CONST_CONSTANT(i)       ((ARB_FFP_CONST_SPECULAR_ENABLE) + 1 + i)
 #define ARB_FFP_CONST_BUMPMAT(i)        ((ARB_FFP_CONST_CONSTANT(7)) + 1 + i)
 #define ARB_FFP_CONST_LUMINANCE(i)      ((ARB_FFP_CONST_BUMPMAT(7)) + 1 + i)
@@ -6030,7 +6031,7 @@ static void color_key_arbfp(struct wined3d_context *context, const struct wined3
 {
     struct wined3d_device *device = context->swapchain->device;
     const struct wined3d_gl_info *gl_info = context->gl_info;
-    struct wined3d_color float_key;
+    struct wined3d_color float_key[2];
     const struct wined3d_texture *texture = state->textures[0];
 
     if (!texture)
@@ -6046,14 +6047,17 @@ static void color_key_arbfp(struct wined3d_context *context, const struct wined3
             return;
 
         priv = device->shader_priv;
-        priv->pshader_const_dirty[ARB_FFP_CONST_COLOR_KEY] = 1;
-        priv->highest_dirty_ps_const = max(priv->highest_dirty_ps_const, ARB_FFP_CONST_COLOR_KEY + 1);
+        priv->pshader_const_dirty[ARB_FFP_CONST_COLOR_KEY_LOW] = 1;
+        priv->pshader_const_dirty[ARB_FFP_CONST_COLOR_KEY_HIGH] = 1;
+        priv->highest_dirty_ps_const = max(priv->highest_dirty_ps_const, ARB_FFP_CONST_COLOR_KEY_HIGH + 1);
     }
 
-    wined3d_format_convert_color_to_float(texture->resource.format, NULL,
-            texture->async.src_blt_color_key.color_space_high_value, &float_key);
-    GL_EXTCALL(glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY, &float_key.r));
-    checkGLcall("glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY, &float_key.r)");
+    wined3d_format_get_float_color_key(texture->resource.format, &texture->async.src_blt_color_key, float_key);
+
+    GL_EXTCALL(glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY_LOW, &float_key[0].r));
+    checkGLcall("glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY_LOW, &float_key[0].r)");
+    GL_EXTCALL(glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY_HIGH, &float_key[1].r));
+    checkGLcall("glProgramEnvParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB, ARB_FFP_CONST_COLOR_KEY_HIGH, &float_key[1].r)");
 }
 
 static const char *get_argreg(struct wined3d_string_buffer *buffer, DWORD argnum, unsigned int stage, DWORD arg)
@@ -6308,7 +6312,8 @@ static GLuint gen_arbfp_ffp_shader(const struct ffp_frag_settings *settings, con
 
     if (settings->color_key_enabled)
     {
-        shader_addline(&buffer, "PARAM color_key = program.env[%u];\n", ARB_FFP_CONST_COLOR_KEY);
+        shader_addline(&buffer, "PARAM color_key_low = program.env[%u];\n", ARB_FFP_CONST_COLOR_KEY_LOW);
+        shader_addline(&buffer, "PARAM color_key_high = program.env[%u];\n", ARB_FFP_CONST_COLOR_KEY_HIGH);
         tex_read[0] = TRUE;
     }
 
@@ -6492,10 +6497,12 @@ static GLuint gen_arbfp_ffp_shader(const struct ffp_frag_settings *settings, con
 
     if (settings->color_key_enabled)
     {
-        shader_addline(&buffer, "SUB TMP, tex0, color_key;\n");
-        shader_addline(&buffer, "DP4 TMP.b, TMP, TMP;\n");
-        shader_addline(&buffer, "SGE TMP, -TMP.b, 0.0;\n");
-        shader_addline(&buffer, "KIL -TMP;\n");
+        shader_addline(&buffer, "SLT TMP, tex0, color_key_low;\n"); /* below low key */
+        shader_addline(&buffer, "SGE ret, tex0, color_key_high;\n"); /* above high key */
+        shader_addline(&buffer, "ADD TMP, TMP, ret;\n"); /* or */
+        shader_addline(&buffer, "DP4 TMP.b, TMP, TMP;\n"); /* on any channel */
+        shader_addline(&buffer, "SGE TMP, -TMP.b, 0.0;\n"); /* logical not */
+        shader_addline(&buffer, "KIL -TMP;\n"); /* discard if true */
     }
 
     /* Generate the main shader */
@@ -6901,7 +6908,8 @@ struct arbfp_blit_desc
 };
 
 #define ARBFP_BLIT_PARAM_SIZE 0
-#define ARBFP_BLIT_PARAM_COLOR_KEY 1
+#define ARBFP_BLIT_PARAM_COLOR_KEY_LOW 1
+#define ARBFP_BLIT_PARAM_COLOR_KEY_HIGH 2
 
 struct arbfp_blit_priv
 {
@@ -7586,13 +7594,16 @@ static GLuint arbfp_gen_plain_shader(struct arbfp_blit_priv *priv,
     if (type->use_color_key)
     {
         shader_addline(&buffer, "TEMP color;\n");
-        shader_addline(&buffer, "TEMP compare;\n");
-        shader_addline(&buffer, "PARAM color_key = program.local[%u];\n", ARBFP_BLIT_PARAM_COLOR_KEY);
+        shader_addline(&buffer, "TEMP less, greater;\n");
+        shader_addline(&buffer, "PARAM color_key_low = program.local[%u];\n", ARBFP_BLIT_PARAM_COLOR_KEY_LOW);
+        shader_addline(&buffer, "PARAM color_key_high = program.local[%u];\n", ARBFP_BLIT_PARAM_COLOR_KEY_HIGH);
         shader_addline(&buffer, "TEX color, fragment.texcoord[0], texture[0], %s;\n", tex_target);
-        shader_addline(&buffer, "SUB compare, color, color_key;\n");
-        shader_addline(&buffer, "DP4 compare.r, compare, compare;\n");
-        shader_addline(&buffer, "SGE compare, -compare.r, 0.0;\n");
-        shader_addline(&buffer, "KIL -compare;\n");
+        shader_addline(&buffer, "SLT less, color, color_key_low;\n"); /* below low key */
+        shader_addline(&buffer, "SGE greater, color, color_key_high;\n"); /* above high key */
+        shader_addline(&buffer, "ADD less, less, greater;\n"); /* or */
+        shader_addline(&buffer, "DP4 less.b, less, less;\n"); /* on any channel */
+        shader_addline(&buffer, "SGE less, -less.b, 0.0;\n"); /* logical not */
+        shader_addline(&buffer, "KIL -less;\n"); /* discard if true */
         shader_addline(&buffer, "MOV result.color, color;\n");
     }
     else
@@ -7622,7 +7633,7 @@ static HRESULT arbfp_blit_set(void *blit_priv, struct wined3d_context *context,
     struct wine_rb_entry *entry;
     struct arbfp_blit_type type;
     struct arbfp_blit_desc *desc;
-    struct wined3d_color float_color_key;
+    struct wined3d_color float_color_key[2];
 
     if (is_complex_fixup(surface->resource.format->color_fixup))
         fixup = get_complex_fixup(surface->resource.format->color_fixup);
@@ -7723,10 +7734,11 @@ err_out:
     checkGLcall("glProgramLocalParameter4fvARB");
     if (type.use_color_key)
     {
-        wined3d_format_convert_color_to_float(surface->resource.format, NULL,
-                color_key->color_space_high_value, &float_color_key);
+        wined3d_format_get_float_color_key(surface->resource.format, color_key, float_color_key);
+        GL_EXTCALL(glProgramLocalParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB,
+                ARBFP_BLIT_PARAM_COLOR_KEY_LOW, &float_color_key[0].r));
         GL_EXTCALL(glProgramLocalParameter4fvARB(GL_FRAGMENT_PROGRAM_ARB,
-                ARBFP_BLIT_PARAM_COLOR_KEY, &float_color_key.r));
+                ARBFP_BLIT_PARAM_COLOR_KEY_HIGH, &float_color_key[1].r));
         checkGLcall("glProgramLocalParameter4fvARB");
     }
 

dlls/wined3d/glsl_shader.c

@@ -1266,12 +1266,11 @@ static void shader_glsl_load_fog_uniform(const struct wined3d_context *context,
 static void shader_glsl_load_color_key_constant(const struct glsl_ps_program *ps,
         const struct wined3d_gl_info *gl_info, const struct wined3d_state *state)
 {
-    struct wined3d_color float_key;
+    struct wined3d_color float_key[2];
     const struct wined3d_texture *texture = state->textures[0];
 
-    wined3d_format_convert_color_to_float(texture->resource.format, NULL,
-            texture->async.src_blt_color_key.color_space_high_value, &float_key);
-    GL_EXTCALL(glUniform4fv(ps->color_key_location, 1, &float_key.r));
+    wined3d_format_get_float_color_key(texture->resource.format, &texture->async.src_blt_color_key, float_key);
+    GL_EXTCALL(glUniform4fv(ps->color_key_location, 2, &float_key[0].r));
 }
 
 /* Context activation is done by the caller (state handler). */
@@ -6168,7 +6167,7 @@ static GLuint shader_glsl_generate_ffp_fragment_shader(struct shader_glsl_priv *
     if (tfactor_used)
         shader_addline(buffer, "uniform vec4 tex_factor;\n");
     if (settings->color_key_enabled)
-        shader_addline(buffer, "uniform vec4 color_key;\n");
+        shader_addline(buffer, "uniform vec4 color_key[2];\n");
     shader_addline(buffer, "uniform vec4 specular_enable;\n");
 
     if (settings->sRGB_write)
@@ -6332,7 +6331,10 @@ static GLuint shader_glsl_generate_ffp_fragment_shader(struct shader_glsl_priv *
     }
 
     if (settings->color_key_enabled)
-        shader_addline(buffer, "if (all(equal(tex0, color_key))) discard;\n");
+    {
+        shader_addline(buffer, "if (all(greaterThanEqual(tex0, color_key[0])) && all(lessThan(tex0, color_key[1])))\n");
+        shader_addline(buffer, "    discard;\n");
+    }
 
     /* Generate the main shader */
     for (stage = 0; stage < MAX_TEXTURES; ++stage)

dlls/wined3d/utils.c

@@ -4207,6 +4207,72 @@ BOOL wined3d_format_convert_color_to_float(const struct wined3d_format *format,
     }
 }
 
+void wined3d_format_get_float_color_key(const struct wined3d_format *format,
+        const struct wined3d_color_key *key, struct wined3d_color *float_colors)
+{
+    struct wined3d_color slop;
+
+    switch (format->id)
+    {
+        case WINED3DFMT_B8G8R8_UNORM:
+        case WINED3DFMT_B8G8R8A8_UNORM:
+        case WINED3DFMT_B8G8R8X8_UNORM:
+        case WINED3DFMT_B5G6R5_UNORM:
+        case WINED3DFMT_B5G5R5X1_UNORM:
+        case WINED3DFMT_B5G5R5A1_UNORM:
+        case WINED3DFMT_B4G4R4A4_UNORM:
+        case WINED3DFMT_B2G3R3_UNORM:
+        case WINED3DFMT_R8_UNORM:
+        case WINED3DFMT_A8_UNORM:
+        case WINED3DFMT_B2G3R3A8_UNORM:
+        case WINED3DFMT_B4G4R4X4_UNORM:
+        case WINED3DFMT_R10G10B10A2_UNORM:
+        case WINED3DFMT_R10G10B10A2_SNORM:
+        case WINED3DFMT_R8G8B8A8_UNORM:
+        case WINED3DFMT_R8G8B8X8_UNORM:
+        case WINED3DFMT_R16G16_UNORM:
+        case WINED3DFMT_B10G10R10A2_UNORM:
+            slop.r = 0.5f / ((1 << format->red_size) - 1);
+            slop.g = 0.5f / ((1 << format->green_size) - 1);
+            slop.b = 0.5f / ((1 << format->blue_size) - 1);
+            slop.a = 0.5f / ((1 << format->alpha_size) - 1);
+
+            float_colors[0].r = color_to_float(key->color_space_low_value, format->red_size, format->red_offset)
+                    - slop.r;
+            float_colors[0].g = color_to_float(key->color_space_low_value, format->green_size, format->green_offset)
+                    - slop.g;
+            float_colors[0].b = color_to_float(key->color_space_low_value, format->blue_size, format->blue_offset)
+                    - slop.b;
+            float_colors[0].a = color_to_float(key->color_space_low_value, format->alpha_size, format->alpha_offset)
+                    - slop.a;
+
+            float_colors[1].r = color_to_float(key->color_space_high_value, format->red_size, format->red_offset)
+                    + slop.r;
+            float_colors[1].g = color_to_float(key->color_space_high_value, format->green_size, format->green_offset)
+                    + slop.g;
+            float_colors[1].b = color_to_float(key->color_space_high_value, format->blue_size, format->blue_offset)
+                    + slop.b;
+            float_colors[1].a = color_to_float(key->color_space_high_value, format->alpha_size, format->alpha_offset)
+                    + slop.a;
+            break;
+
+        case WINED3DFMT_P8_UINT:
+            float_colors[0].r = 0.0f;
+            float_colors[0].g = 0.0f;
+            float_colors[0].b = 0.0f;
+            float_colors[0].a = (key->color_space_low_value - 0.5f) / 255.0f;
+
+            float_colors[1].r = 0.0f;
+            float_colors[1].g = 0.0f;
+            float_colors[1].b = 0.0f;
+            float_colors[1].a = (key->color_space_high_value + 0.5f) / 255.0f;
+            break;
+
+        default:
+            ERR("Unhandled color key to float conversion for format %s.\n", debug_d3dformat(format->id));
+    }
+}
+
 /* DirectDraw stuff */
 enum wined3d_format_id pixelformat_for_depth(DWORD depth)
 {

dlls/wined3d/wined3d_private.h

@@ -3227,6 +3227,8 @@ DWORD wined3d_format_convert_from_float(const struct wined3d_surface *surface,
         const struct wined3d_color *color) DECLSPEC_HIDDEN;
 BOOL wined3d_format_convert_color_to_float(const struct wined3d_format *format,
         const struct wined3d_palette *palette, DWORD color, struct wined3d_color *float_color) DECLSPEC_HIDDEN;
+void wined3d_format_get_float_color_key(const struct wined3d_format *format,
+        const struct wined3d_color_key *key, struct wined3d_color *float_colors) DECLSPEC_HIDDEN;
 const struct wined3d_color_key_conversion * wined3d_format_get_color_key_conversion(
         const struct wined3d_texture *texture, BOOL need_alpha_ck) DECLSPEC_HIDDEN;