From 661ecf3c48e16a9add216287eb969d7615e47968 Mon Sep 17 00:00:00 2001 From: "Marc G. Fournier" Date: Sun, 15 Mar 1998 07:39:04 +0000 Subject: From: t-ishii@sra.co.jp Included are patches intended for allowing PostgreSQL to handle multi-byte charachter sets such as EUC(Extende Unix Code), Unicode and Mule internal code. With the MB patch you can use multi-byte character sets in regexp and LIKE. The encoding system chosen is determined at the compile time. To enable the MB extension, you need to define a variable "MB" in Makefile.global or in Makefile.custom. For further information please take a look at README.mb under doc directory. (Note that unlike "jp patch" I do not use modified GNU regexp any more. I changed Henry Spencer's regexp coming with PostgreSQL.) --- src/backend/regex/utils.c | 348 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 src/backend/regex/utils.c (limited to 'src/backend/regex/utils.c') diff --git a/src/backend/regex/utils.c b/src/backend/regex/utils.c new file mode 100644 index 00000000000..1f904e338ce --- /dev/null +++ b/src/backend/regex/utils.c @@ -0,0 +1,348 @@ +/* + * misc conversion functions between pg_wchar and other encodings. + * Tatsuo Ishii + * $Id: utils.c,v 1.1 1998/03/15 07:38:39 scrappy Exp $ + */ +#include +/* + * convert EUC to pg_wchar (EUC process code) + * caller should allocate enough space for "to" + */ +static void pg_euc2wchar(const unsigned char *from, pg_wchar *to) +{ + while (*from) { + if (*from == SS2) { + from++; + *to = *from++; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + } else { + *to = *from++; + } + to++; + } + *to = 0; +} + +static void pg_eucjp2wchar(const unsigned char *from, pg_wchar *to) +{ + pg_euc2wchar(from,to); +} + +static void pg_euckr2wchar(const unsigned char *from, pg_wchar *to) +{ + pg_euc2wchar(from,to); +} + +static void pg_eucch2wchar(const unsigned char *from, pg_wchar *to) +{ + while (*from) { + if (*from == SS2) { + from++; + *to = 0x3f00 & (*from++ << 8); + *to = *from++; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + } else { + *to = *from++; + } + to++; + } + *to = 0; +} + +static void pg_euccn2wchar(const unsigned char *from, pg_wchar *to) +{ + while (*from) { + if (*from == SS2) { + from++; + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + } else { + *to = *from++; + } + to++; + } + *to = 0; +} + +/* + * convert UTF-8 to pg_wchar (UCS-2) + * caller should allocate enough space for "to" + */ +static void pg_utf2wchar(const unsigned char *from, pg_wchar *to) +{ + unsigned char c1,c2,c3; + while (*from) { + if ((*from & 0x80) == 0) { + *to = *from++; + } else if ((*from & 0xe0) == 0xc0) { + c1 = *from++ & 0x1f; + c2 = *from++ & 0x3f; + *to = c1 << 6; + *to |= c2; + } else if ((*from & 0xe0) == 0xe0) { + c1 = *from++ & 0x0f; + c2 = *from++ & 0x3f; + c3 = *from++ & 0x3f; + *to = c1 << 12; + *to |= c2 << 6; + *to |= c3; + } + to++; + } + *to = 0; +} + +/* + * convert mule internal code to pg_wchar. + * in this case pg_wchar consists of following 4 bytes: + * + * 0x00(unused) + * 0x00(ASCII)|leading character (one of LC1, LC12, LC2 or LC22) + * 0x00(ASCII,1 byte code)|other than 0x00(2 byte code) + * the lowest byte of the code + * + * note that Type N (variable length byte encoding) cannot be represented by + * this schema. sorry. + * caller should allocate enough space for "to" + */ +static void pg_mule2wchar(const unsigned char *from, pg_wchar *to) +{ + while (*from) { + if (IS_LC1(*from)) { + *to = *from++ << 16; + *to |= *from++; + } else if (IS_LCPRV1(*from)) { + from++; + *to = *from++ << 16; + *to |= *from++; + } else if (IS_LC2(*from)) { + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + } else if (IS_LCPRV2(*from)) { + from++; + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + } else { /* assume ASCII */ + *to = *from++; + } + to++; + } + *to = 0; +} + +/* + * convert EUC to pg_wchar (EUC process code) + * caller should allocate enough space for "to" + * len: length of from. + * "from" not necessarily null terminated. + */ +static void pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) +{ + while (*from && len > 0) { + if (*from == SS2) { + from++; + len--; + *to = 0xff & *from++; + len--; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + len -= 3; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + len -= 2; + } else { + *to = *from++; + len--; + } + to++; + } + *to = 0; +} + +static void pg_eucjp2wchar_with_len +(const unsigned char *from, pg_wchar *to, int len) +{ + pg_euc2wchar_with_len(from,to,len); +} + +static void pg_euckr2wchar_with_len +(const unsigned char *from, pg_wchar *to, int len) +{ + pg_euc2wchar_with_len(from,to,len); +} + +static void pg_eucch2wchar_with_len +(const unsigned char *from, pg_wchar *to, int len) +{ + while (*from && len > 0) { + if (*from == SS2) { + from++; + len--; + *to = 0x3f00 & (*from++ << 8); + *to = *from++; + len -= 2; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + len -= 3; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + len -= 2; + } else { + *to = *from++; + len--; + } + to++; + } + *to = 0; +} + +static void pg_euccn2wchar_with_len +(const unsigned char *from, pg_wchar *to, int len) +{ + while (*from && len > 0) { + if (*from == SS2) { + from++; + len--; + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + len -= 3; + } else if (*from == SS3) { + from++; + *to = *from++ << 8; + *to |= 0x3f & *from++; + len -= 3; + } else if (*from & 0x80) { + *to = *from++ << 8; + *to |= *from++; + len -= 2; + } else { + *to = *from++; + len--; + } + to++; + } + *to = 0; +} + +/* + * convert UTF-8 to pg_wchar (UCS-2) + * caller should allocate enough space for "to" + * len: length of from. + * "from" not necessarily null terminated. + */ +static void pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) +{ + unsigned char c1,c2,c3; + while (*from && len > 0) { + if ((*from & 0x80) == 0) { + *to = *from++; + len--; + } else if ((*from & 0xe0) == 0xc0) { + c1 = *from++ & 0x1f; + c2 = *from++ & 0x3f; + len -= 2; + *to = c1 << 6; + *to |= c2; + } else if ((*from & 0xe0) == 0xe0) { + c1 = *from++ & 0x0f; + c2 = *from++ & 0x3f; + c3 = *from++ & 0x3f; + len -= 3; + *to = c1 << 12; + *to |= c2 << 6; + *to |= c3; + } + to++; + } + *to = 0; +} + +/* + * convert mule internal code to pg_wchar + * caller should allocate enough space for "to" + * len: length of from. + * "from" not necessarily null terminated. + */ +static void pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) +{ + while (*from && len > 0) { + if (IS_LC1(*from)) { + *to = *from++ << 16; + *to |= *from++; + len -= 2; + } else if (IS_LCPRV1(*from)) { + from++; + *to = *from++ << 16; + *to |= *from++; + len -= 3; + } else if (IS_LC2(*from)) { + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + len -= 3; + } else if (IS_LCPRV2(*from)) { + from++; + *to = *from++ << 16; + *to |= *from++ << 8; + *to |= *from++; + len -= 4; + } else { /* assume ASCII */ + *to = (unsigned char)*from++; + len--; + } + to++; + } + *to = 0; +} + +typedef struct { + void (*mb2wchar)(); + void (*mb2wchar_with_len)(); +} pg_wchar_tbl; + +static pg_wchar_tbl pg_wchar_table[] = { + {pg_eucjp2wchar, pg_eucjp2wchar_with_len}, + {pg_eucch2wchar, pg_eucch2wchar_with_len}, + {pg_euckr2wchar, pg_euckr2wchar_with_len}, + {pg_euccn2wchar, pg_euccn2wchar_with_len}, + {pg_utf2wchar, pg_utf2wchar_with_len}, + {pg_mule2wchar, pg_mule2wchar_with_len}}; + +void pg_mb2wchar(const unsigned char *from, pg_wchar *to) +{ + (*pg_wchar_table[MB].mb2wchar)(from,to); +} + +void pg_mb2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) +{ + (*pg_wchar_table[MB].mb2wchar_with_len)(from,to,len); +} -- cgit v1.2.3