wined3d: Handle WINED3DFMT_FLAG_BROKEN_PITCH in wined3d_format_calculate_size().

The broken pitch calculation also affects the resource memory layout, as evidenced by the mip-tree layout tests. Found using apitrace, which crashes on Wine when playing back traces containing ATI1 textures. Signed-off-by: Patrick Rudolph <siro@das-labor.org> Signed-off-by: Henri Verbeet <hverbeet@codeweavers.com> Signed-off-by: Alexandre Julliard <julliard@winehq.org>

wined3d: Handle WINED3DFMT_FLAG_BROKEN_PITCH in wined3d_format_calculate_size().
3f7e5d64 · Patrick Rudolph · Alexandre Julliard · 09b09b30 · 3f7e5d64 · 3f7e5d64
Commit 3f7e5d64 authored Aug 26, 2016 by Patrick Rudolph Committed by Alexandre Julliard Aug 26, 2016
Hide whitespace changes
Inline Side-by-side

Showing with 13 additions and 6 deletions

device.c dlls/d3d8/tests/device.c +5 -3

device.c dlls/d3d9/tests/device.c +5 -3

utils.c dlls/wined3d/utils.c +3 -0

No files found.
--- a/dlls/d3d8/tests/device.c
+++ b/dlls/d3d8/tests/device.c
@@ -7924,9 +7924,11 @@ static void test_miptree_layout(void)
    }
    formats[] =
    {
-        {D3DFMT_A8R8G8B8, "D3DFMT_A8R8G8B8"},
-        {D3DFMT_A8,       "D3DFMT_A8"},
-        {D3DFMT_L8,       "D3DFMT_L8"},
+        {D3DFMT_A8R8G8B8,             "D3DFMT_A8R8G8B8"},
+        {D3DFMT_A8,                   "D3DFMT_A8"},
+        {D3DFMT_L8,                   "D3DFMT_L8"},
+        {MAKEFOURCC('A','T','I','1'), "D3DFMT_ATI1"},
+        {MAKEFOURCC('A','T','I','2'), "D3DFMT_ATI2"},
    };
    static const struct
    {

--- a/dlls/d3d9/tests/device.c
+++ b/dlls/d3d9/tests/device.c
@@ -11046,9 +11046,11 @@ static void test_miptree_layout(void)
    }
    formats[] =
    {
-        {D3DFMT_A8R8G8B8, "D3DFMT_A8R8G8B8"},
-        {D3DFMT_A8,       "D3DFMT_A8"},
-        {D3DFMT_L8,       "D3DFMT_L8"},
+        {D3DFMT_A8R8G8B8,             "D3DFMT_A8R8G8B8"},
+        {D3DFMT_A8,                   "D3DFMT_A8"},
+        {D3DFMT_L8,                   "D3DFMT_L8"},
+        {MAKEFOURCC('A','T','I','1'), "D3DFMT_ATI1"},
+        {MAKEFOURCC('A','T','I','2'), "D3DFMT_ATI2"},
    };
    static const struct
    {

--- a/dlls/wined3d/utils.c
+++ b/dlls/wined3d/utils.c
@@ -3577,6 +3577,9 @@ UINT wined3d_format_calculate_size(const struct wined3d_format *format, UINT ali
    if (format->id == WINED3DFMT_UNKNOWN)
        return 0;

+    if (format->flags[WINED3D_GL_RES_TYPE_TEX_2D] & WINED3DFMT_FLAG_BROKEN_PITCH)
+        return width * height * depth * format->byte_count;
+
    wined3d_format_calculate_pitch(format, alignment, width, height, &row_pitch, &slice_pitch);

    return slice_pitch * depth;