aboutsummaryrefslogtreecommitdiff
path: root/src/utf.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/utf.c')
-rw-r--r--src/utf.c440
1 files changed, 428 insertions, 12 deletions
diff --git a/src/utf.c b/src/utf.c
index 6990553e0..d9c7e96d7 100644
--- a/src/utf.c
+++ b/src/utf.c
@@ -12,7 +12,7 @@
** This file contains routines used to translate between UTF-8,
** UTF-16, UTF-16BE, and UTF-16LE.
**
-** $Id: utf.c,v 1.1 2004/05/04 15:00:47 drh Exp $
+** $Id: utf.c,v 1.2 2004/05/06 23:37:53 danielk1977 Exp $
**
** Notes on UTF-8:
**
@@ -29,15 +29,318 @@
** 110110wwwwxxxxxx 110111yyyyyyyyyy 000uuuuu xxxxxxyy yyyyyyyy
** xxxxxxxxyyyyyyyy 00000000 xxxxxxxx yyyyyyyy
**
+**
** BOM or Byte Order Mark:
** 0xff 0xfe little-endian utf-16 follows
** 0xfe 0xff big-endian utf-16 follows
+**
+**
+** Handling of malformed strings:
+**
+** SQLite accepts and processes malformed strings without an error wherever
+** possible. However this is not possible when converting between UTF-8 and
+** UTF-16.
+**
+** When converting malformed UTF-8 strings to UTF-16, one instance of the
+** replacement character U+FFFD for each byte that cannot be interpeted as
+** part of a valid unicode character.
+**
+** When converting malformed UTF-16 strings to UTF-8, one instance of the
+** replacement character U+FFFD for each pair of bytes that cannot be
+** interpeted as part of a valid unicode character.
+*/
+
+#include <assert.h>
+#include <unistd.h>
+#include "sqliteInt.h"
+
+typedef struct UtfString UtfString;
+struct UtfString {
+ unsigned char *pZ; /* Raw string data */
+ int n; /* Allocated length of pZ in bytes */
+ int c; /* Number of pZ bytes already read or written */
+};
+
+/* TODO: Implement this macro in os.h. It should be 1 on big-endian
+** machines, and 0 on little-endian.
+*/
+#define SQLITE3_NATIVE_BIGENDIAN 0
+
+#if SQLITE3_NATIVE_BIGENDIAN == 1
+#define BOM_BIGENDIAN 0x0000FFFE
+#define BOM_LITTLEENDIAN 0x0000FEFF
+#else
+#define BOM_BIGENDIAN 0x0000FEFF
+#define BOM_LITTLEENDIAN 0x0000FFFE
+#endif
+
+/*
+** These two macros are used to interpret the first two bytes of the
+** unsigned char array pZ as a 16-bit unsigned int. BE16() for a big-endian
+** interpretation, LE16() for little-endian.
+*/
+#define BE16(pZ) (((u16)((pZ)[0])<<8) + (u16)((pZ)[1]))
+#define LE16(pZ) (((u16)((pZ)[1])<<8) + (u16)((pZ)[0]))
+
+/*
+** READ_16 interprets the first two bytes of the unsigned char array pZ
+** as a 16-bit unsigned int. If big_endian is non-zero the intepretation
+** is big-endian, otherwise little-endian.
+*/
+#define READ_16(pZ,big_endian) (big_endian?BE16(pZ):LE16(pZ))
+
+/*
+** Read the BOM from the start of *pStr, if one is present. Return zero
+** for little-endian, non-zero for big-endian. If no BOM is present, return
+** the machines native byte order.
+**
+** Return values:
+** 1 -> big-endian string
+** 0 -> little-endian string
+*/
+static int readUtf16Bom(UtfString *pStr){
+ /* The BOM must be the first thing read from the string */
+ assert( pStr->c==0 );
+
+ /* If the string data consists of 1 byte or less, the BOM will make no
+ ** difference anyway. In this case just fall through to the default case
+ ** and return the native byte-order for this machine.
+ **
+ ** Otherwise, check the first 2 bytes of the string to see if a BOM is
+ ** present.
+ */
+ if( pStr->n>1 ){
+ u32 bom = BE16(pStr->pZ);
+ if( bom==BOM_BIGENDIAN ){
+ pStr->c = 2;
+ return 1;
+ }
+ if( bom==BOM_LITTLEENDIAN ){
+ pStr->c = 2;
+ return 0;
+ }
+ }
+
+ return SQLITE3_NATIVE_BIGENDIAN;
+}
+
+
+/*
+** Read a single unicode character from the UTF-8 encoded string *pStr. The
+** value returned is a unicode scalar value. In the case of malformed
+** strings, the unicode replacement character U+FFFD may be returned.
+*/
+static u32 readUtf8(UtfString *pStr){
+ struct Utf8TblRow {
+ u8 b1_mask;
+ u8 b1_masked_val;
+ u8 b1_value_mask;
+ int trailing_bytes;
+ };
+ static const struct Utf8TblRow utf8tbl[] = {
+ { 0x80, 0x00, 0x7F, 0 },
+ { 0xE0, 0xC0, 0x1F, 1 },
+ { 0xF0, 0xE0, 0x0F, 2 },
+ { 0xF8, 0xF0, 0x0E, 3 },
+ { 0, 0, 0, 0}
+ };
+
+ u8 b1; /* First byte of the potentially multi-byte utf-8 character */
+ u32 ret = 0; /* Return value */
+ int ii;
+ struct Utf8TblRow const *pRow;
+
+ pRow = &(utf8tbl[0]);
+
+ b1 = pStr->pZ[pStr->c];
+ pStr->c++;
+ while( pRow->b1_mask && (b1&pRow->b1_mask)!=pRow->b1_masked_val ){
+ pRow++;
+ }
+ if( !pRow->b1_mask ){
+ return 0xFFFD;
+ }
+
+ ret = (u32)(b1&pRow->b1_value_mask);
+ for( ii=0; ii<pRow->trailing_bytes; ii++ ){
+ u8 b = pStr->pZ[pStr->c+ii];
+ if( (b&0xC0)!=0x80 ){
+ return 0xFFFD;
+ }
+ ret = (ret<<6) + (u32)(b&0x3F);
+ }
+
+ pStr->c += pRow->trailing_bytes;
+ return ret;
+}
+
+/*
+** Write the unicode character 'code' to the string pStr using UTF-8
+** encoding. SQLITE_NOMEM may be returned if sqlite3Malloc() fails.
*/
+static int writeUtf8(UtfString *pStr, u32 code){
+ struct Utf8WriteTblRow {
+ u32 max_code;
+ int trailing_bytes;
+ u8 b1_and_mask;
+ u8 b1_or_mask;
+ };
+ static const struct Utf8WriteTblRow utf8tbl[] = {
+ {0x0000007F, 0, 0x7F, 0x00},
+ {0x000007FF, 1, 0xDF, 0xC0},
+ {0x0000FFFF, 2, 0xEF, 0xE0},
+ {0x0010FFFF, 3, 0xF7, 0xF0},
+ {0x00000000, 0, 0x00, 0x00}
+ };
+ static const struct Utf8WriteTblRow *pRow = &utf8tbl[0];
+
+ while( code<=pRow->max_code ){
+ assert( pRow->max_code );
+ pRow++;
+ }
+
+ /* Ensure there is enough room left in the output buffer to write
+ ** this UTF-8 character.
+ */
+ assert( (pStr->n-pStr->c)>=(pRow->trailing_bytes+1) );
+
+ /* Write the UTF-8 encoded character to pStr. All cases below are
+ ** intentionally fall-through.
+ */
+ switch( pRow->trailing_bytes ){
+ case 3:
+ pStr->pZ[pStr->c+3] = (((u8)code)&0x3F)|0x80;
+ code = code>>6;
+ case 2:
+ pStr->pZ[pStr->c+2] = (((u8)code)&0x3F)|0x80;
+ code = code>>6;
+ case 1:
+ pStr->pZ[pStr->c+1] = (((u8)code)&0x3F)|0x80;
+ code = code>>6;
+ case 0:
+ pStr->pZ[pStr->c] = (((u8)code)&(pRow->b1_and_mask))|(pRow->b1_or_mask);
+ }
+ pStr->c += (pRow->trailing_bytes + 1);
+
+ return 0;
+}
+
+/*
+** Read a single unicode character from the UTF-16 encoded string *pStr. The
+** value returned is a unicode scalar value. In the case of malformed
+** strings, the unicode replacement character U+FFFD may be returned.
+**
+** If big_endian is true, the string is assumed to be UTF-16BE encoded.
+** Otherwise, it is UTF-16LE encoded.
+*/
+static u32 readUtf16(UtfString *pStr, int big_endian){
+ u32 code_point; /* the first code-point in the character */
+
+ /* If there is only one byte of data left in the string, return the
+ ** replacement character.
+ */
+ if( (pStr->n-pStr->c)==1 ){
+ pStr->c++;
+ return (int)0xFFFD;
+ }
+
+ code_point = READ_16(&(pStr->pZ[pStr->c]), big_endian);
+ pStr->c += 2;
+
+ /* If this is a non-surrogate code-point, just cast it to an int and
+ ** return the code-point value.
+ */
+ if( code_point<0xD800 || code_point>0xE000 ){
+ return code_point;
+ }
+
+ /* If this is a trailing surrogate code-point, then the string is
+ ** malformed; return the replacement character.
+ */
+ if( code_point>0xDBFF ){
+ return 0xFFFD;
+ }
+
+ /* The code-point just read is a leading surrogate code-point. If their
+ ** is not enough data left or the next code-point is not a trailing
+ ** surrogate, return the replacement character.
+ */
+ if( (pStr->n-pStr->c)>1 ){
+ u32 code_point2 = READ_16(&pStr->pZ[pStr->c], big_endian);
+ if( code_point2<0xDC00 || code_point>0xDFFF ){
+ return 0xFFFD;
+ }
+ pStr->c += 2;
+
+ return (
+ (((code_point&0x03C0)+0x0040)<<16) + /* uuuuu */
+ ((code_point&0x003F)<<10) + /* xxxxxx */
+ (code_point2&0x03FF) /* yy yyyyyyyy */
+ );
+
+ }else{
+ return (int)0xFFFD;
+ }
+
+ /* not reached */
+}
+
+static int writeUtf16(UtfString *pStr, int code, int big_endian){
+ int bytes;
+ unsigned char *hi_byte;
+ unsigned char *lo_byte;
+
+ bytes = (code>0x0000FFFF?4:2);
+
+ /* Ensure there is enough room left in the output buffer to write
+ ** this UTF-8 character.
+ */
+ assert( (pStr->n-pStr->c)>=bytes );
+
+ /* Initialise hi_byte and lo_byte to point at the locations into which
+ ** the MSB and LSB of the (first) 16-bit unicode code-point written for
+ ** this character.
+ */
+ hi_byte = (big_endian?&pStr->pZ[pStr->c]:&pStr->pZ[pStr->c+1]);
+ lo_byte = (big_endian?&pStr->pZ[pStr->c+1]:&pStr->pZ[pStr->c]);
+
+ if( bytes==2 ){
+ *hi_byte = (u8)((code&0x0000FF00)>>8);
+ *lo_byte = (u8)(code&0x000000FF);
+ }else{
+ u32 wrd;
+ wrd = ((((code&0x001F0000)-0x00010000)+(code&0x0000FC00))>>10)|0x0000D800;
+ *hi_byte = (u8)((wrd&0x0000FF00)>>8);
+ *lo_byte = (u8)(wrd&0x000000FF);
+
+ wrd = (code&0x000003FF)|0x0000DC00;
+ *(hi_byte+2) = (u8)((wrd&0x0000FF00)>>8);
+ *(lo_byte+2) = (u8)(wrd&0x000000FF);
+ }
+
+ pStr->c += bytes;
+
+ return 0;
+}
+
+/*
+** Return the number of bytes up to (but not including) the first \u0000
+** character in *pStr.
+*/
+static int utf16Bytelen(const unsigned char *pZ){
+ const unsigned char *pC1 = pZ;
+ const unsigned char *pC2 = pZ+1;
+ while( *pC1 || *pC2 ){
+ pC1 += 2;
+ pC2 += 2;
+ }
+ return pC1-pZ;
+}
/*
** Convert a string in UTF-16 native byte (or with a Byte-order-mark or
** "BOM") into a UTF-8 string. The UTF-8 string is written into space
-** obtained from sqlit3Malloc() and must be released by the calling function.
+** obtained from sqlite3Malloc() and must be released by the calling function.
**
** The parameter N is the number of bytes in the UTF-16 string. If N is
** negative, the entire string up to the first \u0000 character is translated.
@@ -45,7 +348,113 @@
** The returned UTF-8 string is always \000 terminated.
*/
unsigned char *sqlite3utf16to8(const void *pData, int N){
- unsigned char *in = (unsigned char *)pData;
+ UtfString in;
+ UtfString out;
+ int big_endian;
+
+ out.pZ = 0;
+
+ in.pZ = (unsigned char *)pData;
+ in.n = N;
+ in.c = 0;
+
+ if( in.n<0 ){
+ in.n = utf16Bytelen(in.pZ);
+ }
+
+ /* A UTF-8 encoding of a unicode string can require at most 1.5 times as
+ ** much space to store as the same string encoded using UTF-16. Allocate
+ ** this now.
+ */
+ out.n = (in.n*1.5) + 1;
+ out.pZ = sqliteMalloc(in.n);
+ if( !out.pZ ){
+ return 0;
+ }
+ out.c = 0;
+
+ big_endian = readUtf16Bom(&in);
+ while( in.c<in.n ){
+ writeUtf8(&out, readUtf16(&in, big_endian));
+ }
+
+ /* Add the NULL-terminator character */
+ assert( out.c<out.n );
+ out.pZ[out.c] = 0x00;
+
+ return out.pZ;
+}
+
+static void *utf8toUtf16(const unsigned char *pIn, int N, int big_endian){
+ UtfString in;
+ UtfString out;
+
+ in.pZ = (unsigned char *)pIn;
+ in.n = N;
+ in.c = 0;
+
+ if( in.n<0 ){
+ in.n = strlen(in.pZ);
+ }
+
+ /* A UTF-16 encoding of a unicode string can require at most twice as
+ ** much space to store as the same string encoded using UTF-8. Allocate
+ ** this now.
+ */
+ out.n = (in.n*2) + 2;
+ out.pZ = sqliteMalloc(in.n);
+ if( !out.pZ ){
+ return 0;
+ }
+ out.c = 0;
+
+ while( in.c<in.n ){
+ writeUtf16(&out, readUtf8(&in), big_endian);
+ }
+
+ /* Add the NULL-terminator character */
+ assert( (out.c+1)<out.n );
+ out.pZ[out.c] = 0x00;
+ out.pZ[out.c+1] = 0x00;
+
+ return out.pZ;
+}
+
+/*
+** Translate UTF-8 to UTF-16BE or UTF-16LE
+*/
+void *sqlite3utf8to16be(const unsigned char *pIn, int N){
+ return utf8toUtf16(pIn, N, 1);
+}
+
+void *sqlite3utf8to16le(const unsigned char *pIn, int N){
+ return utf8toUtf16(pIn, N, 0);
+}
+
+/*
+** This routine does the work for sqlite3utf16to16le() and
+** sqlite3utf16to16be(). If big_endian is 1 the input string is
+** transformed in place to UTF-16BE encoding. If big_endian is 0 then
+** the input is transformed to UTF-16LE.
+**
+** Unless the first two bytes of the input string is a BOM, the input is
+** assumed to be UTF-16 encoded using the machines native byte ordering.
+*/
+static void utf16to16(void *pData, int N, int big_endian){
+ UtfString inout;
+ inout.pZ = (unsigned char *)pData;
+ inout.c = 0;
+ inout.n = N;
+
+ if( inout.n<0 ){
+ inout.n = utf16Bytelen(inout.pZ);
+ }
+
+ if( readUtf16Bom(&inout)!=big_endian ){
+ swab(&inout.pZ[inout.c], inout.pZ, inout.n-inout.c);
+ }else if( inout.c ){
+ memmove(inout.pZ, &inout.pZ[inout.c], inout.n-inout.c);
+ }
}
/*
@@ -57,21 +466,28 @@ unsigned char *sqlite3utf16to8(const void *pData, int N){
** If the native byte order is little-endian and there is no BOM, then
** this routine is a no-op. If there is a BOM at the start of the string,
** it is removed.
+**
+** Translation from UTF-16LE to UTF-16BE and back again is accomplished
+** using the library function swab().
*/
void sqlite3utf16to16le(void *pData, int N){
-}
-void sqlite3utf16to16be(void *pData, int N){
+ utf16to16(pData, N, 0);
}
/*
+** Convert a string in UTF-16 native byte or with a BOM into a UTF-16BE
+** string. The conversion occurs in-place. The output overwrites the
+** input. N bytes are converted. If N is negative everything is converted
+** up to the first \u0000 character.
+**
+** If the native byte order is little-endian and there is no BOM, then
+** this routine is a no-op. If there is a BOM at the start of the string,
+** it is removed.
+**
** Translation from UTF-16LE to UTF-16BE and back again is accomplished
** using the library function swab().
*/
-
-/*
-** Translate UTF-8 to UTF-16BE or UTF-16LE
-*/
-void *sqlite3utf8to16be(const unsigned char *pIn, int N){
-}
-void *sqlite3utf8to16le(const unsigned char *pIn, int N){
+void sqlite3utf16to16be(void *pData, int N){
+ utf16to16(pData, N, 1);
}
+
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-19 20:11:16 +0000