diff options
Diffstat (limited to 'src/backend/utils/mb/wchar.c')
| -rw-r--r-- | src/backend/utils/mb/wchar.c | 213 |
1 files changed, 197 insertions, 16 deletions
diff --git a/src/backend/utils/mb/wchar.c b/src/backend/utils/mb/wchar.c index 346668711f1..2f0725363cc 100644 --- a/src/backend/utils/mb/wchar.c +++ b/src/backend/utils/mb/wchar.c @@ -1,7 +1,7 @@ /* * conversion functions between pg_wchar and multibyte streams. * Tatsuo Ishii - * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.52 2005/12/26 19:30:44 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.53 2006/02/10 00:39:04 momjian Exp $ * * WIN1250 client encoding updated by Pavel Behal * @@ -23,6 +23,13 @@ * for the particular encoding. Note that if the encoding is only * supported in the client, you don't need to define * mb2wchar_with_len() function (SJIS is the case). + * + * Note: for the display output of psql to work properly, the return values + * of these functions must conform to the Unicode standard. In particular + * the NUL character is zero width and control characters are generally + * width -1. It is recommended that non-ASCII encodings refer their ASCII + * subset to the ASCII routines to ensure consistancy. + * */ /* @@ -53,6 +60,11 @@ pg_ascii_mblen(const unsigned char *s) static int pg_ascii_dsplen(const unsigned char *s) { + if (*s == '\0') + return 0; + if (*s < 0x20 || *s == 0x7f) + return -1; + return 1; } @@ -125,7 +137,7 @@ pg_euc_dsplen(const unsigned char *s) else if (IS_HIGHBIT_SET(*s)) len = 2; else - len = 1; + len = pg_ascii_dsplen(s); return len; } @@ -156,7 +168,7 @@ pg_eucjp_dsplen(const unsigned char *s) else if (IS_HIGHBIT_SET(*s)) len = 2; else - len = 1; + len = pg_ascii_dsplen(s); return len; } @@ -244,7 +256,7 @@ pg_euccn_dsplen(const unsigned char *s) if (IS_HIGHBIT_SET(*s)) len = 2; else - len = 1; + len = pg_ascii_dsplen(s); return len; } @@ -304,7 +316,7 @@ pg_euctw_mblen(const unsigned char *s) else if (IS_HIGHBIT_SET(*s)) len = 2; else - len = 1; + len = pg_ascii_dsplen(s); return len; } @@ -320,7 +332,7 @@ pg_euctw_dsplen(const unsigned char *s) else if (IS_HIGHBIT_SET(*s)) len = 2; else - len = 1; + len = pg_ascii_dsplen(s); return len; } @@ -419,10 +431,179 @@ pg_utf_mblen(const unsigned char *s) return len; } +/* + * This is an implementation of wcwidth() and wcswidth() as defined in + * "The Single UNIX Specification, Version 2, The Open Group, 1997" + * <http://www.UNIX-systems.org/online.html> + * + * Markus Kuhn -- 2001-09-08 -- public domain + * + * customised for PostgreSQL + * + * original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c + */ + +struct mbinterval +{ + unsigned short first; + unsigned short last; +}; + +/* auxiliary function for binary search in interval table */ +static int +mbbisearch(pg_wchar ucs, const struct mbinterval *table, int max) +{ + int min = 0; + int mid; + + if (ucs < table[0].first || ucs > table[max].last) + return 0; + while (max >= min) + { + mid = (min + max) / 2; + if (ucs > table[mid].last) + min = mid + 1; + else if (ucs < table[mid].first) + max = mid - 1; + else + return 1; + } + + return 0; +} + + +/* The following functions define the column width of an ISO 10646 + * character as follows: + * + * - The null character (U+0000) has a column width of 0. + * + * - Other C0/C1 control characters and DEL will lead to a return + * value of -1. + * + * - Non-spacing and enclosing combining characters (general + * category code Mn or Me in the Unicode database) have a + * column width of 0. + * + * - Other format characters (general category code Cf in the Unicode + * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. + * + * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) + * have a column width of 0. + * + * - Spacing characters in the East Asian Wide (W) or East Asian + * FullWidth (F) category as defined in Unicode Technical + * Report #11 have a column width of 2. + * + * - All remaining characters (including all printable + * ISO 8859-1 and WGL4 characters, Unicode control characters, + * etc.) have a column width of 1. + * + * This implementation assumes that wchar_t characters are encoded + * in ISO 10646. + */ + +static int +ucs_wcwidth(pg_wchar ucs) +{ + /* sorted list of non-overlapping intervals of non-spacing characters */ + static const struct mbinterval combining[] = { + {0x0300, 0x034E}, {0x0360, 0x0362}, {0x0483, 0x0486}, + {0x0488, 0x0489}, {0x0591, 0x05A1}, {0x05A3, 0x05B9}, + {0x05BB, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2}, + {0x05C4, 0x05C4}, {0x064B, 0x0655}, {0x0670, 0x0670}, + {0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED}, + {0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A}, + {0x07A6, 0x07B0}, {0x0901, 0x0902}, {0x093C, 0x093C}, + {0x0941, 0x0948}, {0x094D, 0x094D}, {0x0951, 0x0954}, + {0x0962, 0x0963}, {0x0981, 0x0981}, {0x09BC, 0x09BC}, + {0x09C1, 0x09C4}, {0x09CD, 0x09CD}, {0x09E2, 0x09E3}, + {0x0A02, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42}, + {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A70, 0x0A71}, + {0x0A81, 0x0A82}, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5}, + {0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD}, {0x0B01, 0x0B01}, + {0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F}, {0x0B41, 0x0B43}, + {0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B82, 0x0B82}, + {0x0BC0, 0x0BC0}, {0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40}, + {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, + {0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD}, + {0x0D41, 0x0D43}, {0x0D4D, 0x0D4D}, {0x0DCA, 0x0DCA}, + {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0E31, 0x0E31}, + {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0EB1, 0x0EB1}, + {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD}, + {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37}, + {0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84}, + {0x0F86, 0x0F87}, {0x0F90, 0x0F97}, {0x0F99, 0x0FBC}, + {0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1032}, + {0x1036, 0x1037}, {0x1039, 0x1039}, {0x1058, 0x1059}, + {0x1160, 0x11FF}, {0x17B7, 0x17BD}, {0x17C6, 0x17C6}, + {0x17C9, 0x17D3}, {0x180B, 0x180E}, {0x18A9, 0x18A9}, + {0x200B, 0x200F}, {0x202A, 0x202E}, {0x206A, 0x206F}, + {0x20D0, 0x20E3}, {0x302A, 0x302F}, {0x3099, 0x309A}, + {0xFB1E, 0xFB1E}, {0xFE20, 0xFE23}, {0xFEFF, 0xFEFF}, + {0xFFF9, 0xFFFB} + }; + + /* test for 8-bit control characters */ + if (ucs == 0) + return 0; + + if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff) + return -1; + + /* binary search in table of non-spacing characters */ + if (mbbisearch(ucs, combining, + sizeof(combining) / sizeof(struct mbinterval) - 1)) + return 0; + + /* + * if we arrive here, ucs is not a combining or C0/C1 control character + */ + + return 1 + + (ucs >= 0x1100 && + (ucs <= 0x115f || /* Hangul Jamo init. consonants */ + (ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a && + ucs != 0x303f) || /* CJK ... Yi */ + (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ + (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility + * Ideographs */ + (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ + (ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */ + (ucs >= 0xffe0 && ucs <= 0xffe6) || + (ucs >= 0x20000 && ucs <= 0x2ffff))); +} + +static pg_wchar +utf2ucs(const unsigned char *c) +{ + /* + * one char version of pg_utf2wchar_with_len. no control here, c must + * point to a large enough string + */ + if ((*c & 0x80) == 0) + return (pg_wchar) c[0]; + else if ((*c & 0xe0) == 0xc0) + return (pg_wchar) (((c[0] & 0x1f) << 6) | + (c[1] & 0x3f)); + else if ((*c & 0xf0) == 0xe0) + return (pg_wchar) (((c[0] & 0x0f) << 12) | + ((c[1] & 0x3f) << 6) | + (c[2] & 0x3f)); + else if ((*c & 0xf0) == 0xf0) + return (pg_wchar) (((c[0] & 0x07) << 18) | + ((c[1] & 0x3f) << 12) | + ((c[2] & 0x3f) << 6) | + (c[3] & 0x3f)); + else + /* that is an invalid code on purpose */ + return 0xffffffff; +} + static int pg_utf_dsplen(const unsigned char *s) { - return 1; /* XXX fix me! */ + return ucs_wcwidth(utf2ucs(s)); } /* @@ -499,7 +680,7 @@ pg_mule_mblen(const unsigned char *s) static int pg_mule_dsplen(const unsigned char *s) { - return 1; /* XXX fix me! */ + return pg_ascii_dsplen(s); /* XXX fix me! */ } /* @@ -529,7 +710,7 @@ pg_latin1_mblen(const unsigned char *s) static int pg_latin1_dsplen(const unsigned char *s) { - return 1; + return pg_ascii_dsplen(s); } /* @@ -559,7 +740,7 @@ pg_sjis_dsplen(const unsigned char *s) else if (IS_HIGHBIT_SET(*s)) len = 2; /* kanji? */ else - len = 1; /* should be ASCII */ + len = pg_ascii_dsplen(s); /* should be ASCII */ return len; } @@ -586,7 +767,7 @@ pg_big5_dsplen(const unsigned char *s) if (IS_HIGHBIT_SET(*s)) len = 2; /* kanji? */ else - len = 1; /* should be ASCII */ + len = pg_ascii_dsplen(s); /* should be ASCII */ return len; } @@ -613,7 +794,7 @@ pg_gbk_dsplen(const unsigned char *s) if (IS_HIGHBIT_SET(*s)) len = 2; /* kanji? */ else - len = 1; /* should be ASCII */ + len = pg_ascii_dsplen(s); /* should be ASCII */ return len; } @@ -640,7 +821,7 @@ pg_uhc_dsplen(const unsigned char *s) if (IS_HIGHBIT_SET(*s)) len = 2; /* 2byte? */ else - len = 1; /* should be ASCII */ + len = pg_ascii_dsplen(s); /* should be ASCII */ return len; } @@ -672,10 +853,10 @@ pg_gb18030_dsplen(const unsigned char *s) { int len; - if (!IS_HIGHBIT_SET(*s)) - len = 1; /* ASCII */ - else + if (IS_HIGHBIT_SET(*s)) len = 2; + else + len = pg_ascii_dsplen(s); /* ASCII */ return len; } |