unicode: Add support for high Unicode planes in composition tables.

dlls/kernelbase/locale.c

@@ -42,7 +42,7 @@ WINE_DEFAULT_DEBUG_CHANNEL(nls);
 #define CALINFO_MAX_YEAR 2029
 
 extern UINT CDECL __wine_get_unix_codepage(void);
-extern WCHAR wine_compose( const WCHAR *str ) DECLSPEC_HIDDEN;
+extern unsigned int wine_compose( unsigned int ch1, unsigned int ch2 ) DECLSPEC_HIDDEN;
 
 extern const unsigned short wctype_table[] DECLSPEC_HIDDEN;
 extern const unsigned int collation_table[] DECLSPEC_HIDDEN;
@@ -1628,18 +1628,24 @@ static int wcstombs_dbcs( const CPTABLEINFO *info, const WCHAR *src, unsigned in
 }
 
 
-static inline int is_valid_sbcs_mapping( const CPTABLEINFO *info, DWORD flags,
-                                         WCHAR wch, unsigned char ch )
+static inline int is_valid_sbcs_mapping( const CPTABLEINFO *info, DWORD flags, unsigned int wch )
 {
-    if ((flags & WC_NO_BEST_FIT_CHARS) || ch == info->DefaultChar)
-        return (info->MultiByteTable[ch] == wch);
+    const unsigned char *table = info->WideCharTable;
+
+    if (wch >= 0x10000) return 0;
+    if ((flags & WC_NO_BEST_FIT_CHARS) || table[wch] == info->DefaultChar)
+        return (info->MultiByteTable[table[wch]] == wch);
     return 1;
 }
 
 
-static inline int is_valid_dbcs_mapping( const CPTABLEINFO *info, DWORD flags,
-                                         WCHAR wch, unsigned short ch )
+static inline int is_valid_dbcs_mapping( const CPTABLEINFO *info, DWORD flags, unsigned int wch )
 {
+    const unsigned short *table = info->WideCharTable;
+    unsigned short ch;
+
+    if (wch >= 0x10000) return 0;
+    ch = table[wch];
     if ((flags & WC_NO_BEST_FIT_CHARS) || ch == info->DefaultChar)
     {
         if (ch >> 8) return info->DBCSOffsets[info->DBCSOffsets[ch >> 8] + (ch & 0xff)] == wch;
@@ -1656,7 +1662,8 @@ static int wcstombs_sbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
     const char def = defchar ? *defchar : (char)info->DefaultChar;
     int i;
     BOOL tmp;
-    WCHAR wch, composed;
+    WCHAR wch;
+    unsigned int composed;
 
     if (!used) used = &tmp;  /* avoid checking on every char */
     *used = FALSE;
@@ -1666,10 +1673,10 @@ static int wcstombs_sbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
         for (i = 0; srclen; i++, src++, srclen--)
         {
             wch = *src;
-            if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src )))
+            if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src[0], src[1] )))
             {
                 /* now check if we can use the composed char */
-                if (is_valid_sbcs_mapping( info, flags, composed, table[composed] ))
+                if (is_valid_sbcs_mapping( info, flags, composed ))
                 {
                     /* we have a good mapping, use it */
                     src++;
@@ -1691,7 +1698,7 @@ static int wcstombs_sbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
                 }
                 /* WC_SEPCHARS is the default */
             }
-            if (!*used) *used = !is_valid_sbcs_mapping( info, flags, wch, table[wch] );
+            if (!*used) *used = !is_valid_sbcs_mapping( info, flags, wch );
         }
         return i;
     }
@@ -1699,13 +1706,13 @@ static int wcstombs_sbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
     for (i = dstlen; srclen && i; dst++, i--, src++, srclen--)
     {
         wch = *src;
-        if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src )))
+        if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src[0], src[1] )))
         {
             /* now check if we can use the composed char */
-            *dst = table[composed];
-            if (is_valid_sbcs_mapping( info, flags, composed, table[composed] ))
+            if (is_valid_sbcs_mapping( info, flags, composed ))
             {
                 /* we have a good mapping, use it */
+                *dst = table[composed];
                 src++;
                 srclen--;
                 continue;
@@ -1728,7 +1735,7 @@ static int wcstombs_sbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
         }
 
         *dst = table[wch];
-        if (!is_valid_sbcs_mapping( info, flags, wch, table[wch] ))
+        if (!is_valid_sbcs_mapping( info, flags, wch ))
         {
             *dst = def;
             *used = TRUE;
@@ -1747,7 +1754,8 @@ static int wcstombs_dbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
                                char *dst, unsigned int dstlen, const char *defchar, BOOL *used )
 {
     const USHORT *table = info->WideCharTable;
-    WCHAR wch, composed, defchar_value;
+    WCHAR wch, defchar_value;
+    unsigned int composed;
     unsigned short res;
     BOOL tmp;
     int i;
@@ -1768,13 +1776,13 @@ static int wcstombs_dbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
         for (i = 0; srclen; srclen--, src++, i++)
         {
             wch = *src;
-            if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src )))
+            if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src[0], src[1] )))
             {
                 /* now check if we can use the composed char */
-                res = table[composed];
-                if (is_valid_dbcs_mapping( info, flags, composed, res ))
+                if (is_valid_dbcs_mapping( info, flags, composed ))
                 {
                     /* we have a good mapping for the composed char, use it */
+                    res = table[composed];
                     if (res & 0xff00) i++;
                     src++;
                     srclen--;
@@ -1798,7 +1806,7 @@ static int wcstombs_dbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
             }
 
             res = table[wch];
-            if (!is_valid_dbcs_mapping( info, flags, wch, res ))
+            if (!is_valid_dbcs_mapping( info, flags, wch ))
             {
                 res = defchar_value;
                 *used = TRUE;
@@ -1812,14 +1820,13 @@ static int wcstombs_dbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
     for (i = dstlen; srclen && i; i--, srclen--, src++)
     {
         wch = *src;
-        if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src )))
+        if ((flags & WC_COMPOSITECHECK) && (srclen > 1) && (composed = wine_compose( src[0], src[1] )))
         {
             /* now check if we can use the composed char */
-            res = table[composed];
-
-            if (is_valid_dbcs_mapping( info, flags, composed, res ))
+            if (is_valid_dbcs_mapping( info, flags, composed ))
             {
                 /* we have a good mapping for the composed char, use it */
+                res = table[composed];
                 src++;
                 srclen--;
                 goto output_char;
@@ -1842,7 +1849,7 @@ static int wcstombs_dbcs_slow( const CPTABLEINFO *info, DWORD flags, const WCHAR
         }
 
         res = table[wch];
-        if (!is_valid_dbcs_mapping( info, flags, wch, res ))
+        if (!is_valid_dbcs_mapping( info, flags, wch ))
         {
             res = defchar_value;
             *used = TRUE;

dlls/ntdll/locale.c

@@ -83,7 +83,7 @@ static NLSTABLEINFO nls_info;
 static HMODULE kernel32_handle;
 static CPTABLEINFO unix_table;
 
-extern WCHAR wine_compose( const WCHAR *str ) DECLSPEC_HIDDEN;
+extern unsigned int wine_compose( unsigned int ch1, unsigned int ch2 ) DECLSPEC_HIDDEN;
 extern const unsigned short combining_class_table[] DECLSPEC_HIDDEN;
 extern const unsigned short nfd_table[] DECLSPEC_HIDDEN;
 extern const unsigned short nfkd_table[] DECLSPEC_HIDDEN;
@@ -301,39 +301,43 @@ static NTSTATUS decompose_string( int compat, const WCHAR *src, int src_len, WCH
 }
 
 
-static BOOL is_blocked( WCHAR *starter, WCHAR *ptr )
+static unsigned int compose_string( WCHAR *str, unsigned int srclen )
 {
-    if (ptr == starter + 1) return FALSE;
-    /* Because the string is already canonically ordered, the chars are blocked
-       only if the previous char's combining class is equal to the test char. */
-    if (get_combining_class( *(ptr - 1) ) == get_combining_class( *ptr )) return TRUE;
-    return FALSE;
-}
-
-
-static unsigned int compose_string( WCHAR *str, unsigned int len )
-{
-    unsigned int i, last_starter = len;
-    WCHAR pair[2], comp;
+    unsigned int i, ch, comp, len, start_ch = 0, last_starter = srclen;
+    BYTE class, prev_class = 0;
 
-    for (i = 0; i < len; i++)
+    for (i = 0; i < srclen; i += len)
     {
-        pair[1] = str[i];
-        if (last_starter == len || is_blocked( str + last_starter, str + i ) || !(comp = wine_compose( pair )))
+        if (!(len = get_utf16( str + i, srclen - i, &ch ))) return 0;
+        class = get_combining_class( ch );
+        if (last_starter == srclen || (prev_class && prev_class >= class) ||
+            !(comp = wine_compose( start_ch, ch )))
         {
-            if (is_starter( str[i] ))
+            if (!class)
             {
                 last_starter = i;
-                pair[0] = str[i];
+                start_ch = ch;
             }
-            continue;
+            prev_class = class;
+        }
+        else
+        {
+            int comp_len = 1 + (comp >= 0x10000);
+            int start_len = 1 + (start_ch >= 0x10000);
+
+            if (comp_len != start_len)
+                memmove( str + last_starter + comp_len, str + last_starter + start_len,
+                         (i - (last_starter + start_len)) * sizeof(WCHAR) );
+            memmove( str + i + comp_len - start_len, str + i + len, (srclen - i - len) * sizeof(WCHAR) );
+            srclen += comp_len - start_len - len;
+            start_ch = comp;
+            i = last_starter;
+            len = comp_len;
+            prev_class = 0;
+            put_utf16( str + i, comp );
         }
-        str[last_starter] = pair[0] = comp;
-        len--;
-        memmove( str + i, str + i + 1, (len - i) * sizeof(WCHAR) );
-        i = last_starter;
     }
-    return len;
+    return srclen;
 }
 
 

tools/make_unicode

@@ -2020,7 +2020,7 @@ sub dump_compose_table($)
     # count how many different second chars we have
 
     my $count = 0;
-    for (my $i = 0; $i < 65536; $i++)
+    for (my $i = 0; $i <= $MAX_CHAR; $i++)
     {
         next unless defined $filled[$i];
         $count++;
@@ -2030,7 +2030,7 @@ sub dump_compose_table($)
 
     my $pos = $count + 1;
     my @table = ();
-    for (my $i = 0; $i < 65536; $i++)
+    for (my $i = 0; $i <= $MAX_CHAR; $i++)
     {
         next unless defined $filled[$i];
         push @table, $i, $pos;
@@ -2038,12 +2038,12 @@ sub dump_compose_table($)
     }
     # terminator with last position
     push @table, 0, $pos;
-    printf OUTPUT "static const WCHAR table[0x%x] =\n{\n", 2*$pos;
-    printf OUTPUT "    /* second chars + offsets */\n%s", dump_array( 16, 0, @table );
+    printf OUTPUT "static const unsigned int table[%u] =\n{\n", 2*$pos;
+    printf OUTPUT "    /* second chars + offsets */\n%s", dump_array( 20, 0, @table );
 
     # build the table of first chars and mappings
 
-    for (my $i = 0; $i < 65536; $i++)
+    for (my $i = 0; $i <= $MAX_CHAR; $i++)
     {
         next unless defined $filled[$i];
         my @table = ();
@@ -2052,11 +2052,11 @@ sub dump_compose_table($)
         {
             push @table, $list[$j][0], $list[$j][1];
         }
-        printf OUTPUT ",\n    /* 0x%04x */\n%s", $i, dump_array( 16, 0, @table );
+        printf OUTPUT ",\n    /* 0x%04x */\n%s", $i, dump_array( 20, 0, @table );
     }
     print OUTPUT "\n};\n\n";
     print OUTPUT <<"EOF";
-static inline int binary_search( WCHAR ch, int low, int high )
+static inline int binary_search( unsigned int ch, int low, int high )
 {
     while (low <= high)
     {
@@ -2068,16 +2068,13 @@ static inline int binary_search( WCHAR ch, int low, int high )
     return -1;
 }
 
-WCHAR DECLSPEC_HIDDEN wine_compose( const WCHAR *str )
+unsigned int DECLSPEC_HIDDEN wine_compose( unsigned int ch1, unsigned int ch2 )
 {
-    int pos, idx = 1, start = 0, count = $count;
-    for (;;)
-    {
-        if ((pos = binary_search( str[idx], start, count - 1 )) == -1) return 0;
-        if (!idx--) return table[2 * pos + 1];
-        start = table[2 * pos + 1];
-        count = table[2 * pos + 3];
-    }
+    int pos;
+
+    if ((pos = binary_search( ch2, 0, $count - 1 )) == -1) return 0;
+    if ((pos = binary_search( ch1, table[2 * pos + 1], table[2 * pos + 3] - 1 )) == -1) return 0;
+    return table[2 * pos + 1];
 }
 EOF
     close OUTPUT;