diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/func.c | 66 | ||||
| -rw-r--r-- | src/sqliteInt.h | 61 | ||||
| -rw-r--r-- | src/utf.c | 136 |
3 files changed, 118 insertions, 145 deletions
diff --git a/src/func.c b/src/func.c index 2b51f2328..f33da23eb 100644 --- a/src/func.c +++ b/src/func.c @@ -16,7 +16,7 @@ ** sqliteRegisterBuildinFunctions() found at the bottom of the file. ** All other code has file scope. ** -** $Id: func.c,v 1.156 2007/05/15 01:13:47 drh Exp $ +** $Id: func.c,v 1.157 2007/05/15 11:55:09 drh Exp $ */ #include "sqliteInt.h" #include <ctype.h> @@ -103,7 +103,11 @@ static void lengthFunc( case SQLITE_TEXT: { const unsigned char *z = sqlite3_value_text(argv[0]); if( z==0 ) return; - for(len=0; *z; z++){ if( (0xc0&*z)!=0x80 ) len++; } + len = 0; + while( *z ){ + len++; + SQLITE_SKIP_UTF8(z); + } sqlite3_result_int(context, len); break; } @@ -162,7 +166,6 @@ static void substrFunc( ){ const unsigned char *z; const unsigned char *z2; - int i; int len; int p0type; i64 p1, p2; @@ -176,7 +179,10 @@ static void substrFunc( }else{ z = sqlite3_value_text(argv[0]); if( z==0 ) return; - for(len=0, z2=z; *z2; z2++){ if( (0xc0&*z2)!=0x80 ) len++; } + len = 0; + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } } p1 = sqlite3_value_int(argv[1]); p2 = sqlite3_value_int(argv[2]); @@ -193,16 +199,14 @@ static void substrFunc( p2 = len-p1; } if( p0type!=SQLITE_BLOB ){ - for(i=0; i<p1 && z[i]; i++){ - if( (z[i]&0xc0)==0x80 ) p1++; + while( *z && p1 ){ + SQLITE_SKIP_UTF8(z); + p1--; } - while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p1++; } - for(; i<p1+p2 && z[i]; i++){ - if( (z[i]&0xc0)==0x80 ) p2++; + for(z2=z; *z2 && p2; p2--){ + SQLITE_SKIP_UTF8(z2); } - while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p2++; } - if( p2<0 ) p2 = 0; - sqlite3_result_text(context, (char*)&z[p1], p2, SQLITE_TRANSIENT); + sqlite3_result_text(context, (char*)z, z2-z, SQLITE_TRANSIENT); }else{ if( p2<0 ) p2 = 0; sqlite3_result_blob(context, (char*)&z[p1], p2, SQLITE_TRANSIENT); @@ -389,13 +393,13 @@ static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 }; static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* -** X is a pointer to the first byte of a UTF-8 character. Increment -** X so that it points to the next character. This only works right -** if X points to a well-formed UTF-8 string. +** Read a single UTF-8 character and return its value. */ -#define sqliteNextChar(X) while( (0xc0&*++(X))==0x80 ){} -#define sqliteCharVal(X) sqlite3ReadUtf8(X) - +u32 sqlite3ReadUtf8(const unsigned char *z){ + u32 c; + SQLITE_READ_UTF8(z, c); + return c; +} /* ** Compare two UTF-8 strings for equality where the first string can @@ -446,20 +450,20 @@ static int patternCompare( while( (c=zPattern[1]) == matchAll || c == matchOne ){ if( c==matchOne ){ if( *zString==0 ) return 0; - sqliteNextChar(zString); + SQLITE_SKIP_UTF8(zString); } zPattern++; } if( c && esc && sqlite3ReadUtf8(&zPattern[1])==esc ){ u8 const *zTemp = &zPattern[1]; - sqliteNextChar(zTemp); + SQLITE_SKIP_UTF8(zTemp); c = *zTemp; } if( c==0 ) return 1; if( c==matchSet ){ assert( esc==0 ); /* This is GLOB, not LIKE */ while( *zString && patternCompare(&zPattern[1],zString,pInfo,esc)==0 ){ - sqliteNextChar(zString); + SQLITE_SKIP_UTF8(zString); } return *zString!=0; }else{ @@ -473,20 +477,20 @@ static int patternCompare( } if( c2==0 ) return 0; if( patternCompare(&zPattern[1],zString,pInfo,esc) ) return 1; - sqliteNextChar(zString); + SQLITE_SKIP_UTF8(zString); } return 0; } }else if( !prevEscape && c==matchOne ){ if( *zString==0 ) return 0; - sqliteNextChar(zString); + SQLITE_SKIP_UTF8(zString); zPattern++; }else if( c==matchSet ){ int prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; - c = sqliteCharVal(zString); + c = sqlite3ReadUtf8(zString); if( c==0 ) return 0; c2 = *++zPattern; if( c2=='^' ){ invert = 1; c2 = *++zPattern; } @@ -494,10 +498,10 @@ static int patternCompare( if( c==']' ) seen = 1; c2 = *++zPattern; } - while( (c2 = sqliteCharVal(zPattern))!=0 && c2!=']' ){ + while( (c2 = sqlite3ReadUtf8(zPattern))!=0 && c2!=']' ){ if( c2=='-' && zPattern[1]!=']' && zPattern[1]!=0 && prior_c>0 ){ zPattern++; - c2 = sqliteCharVal(zPattern); + c2 = sqlite3ReadUtf8(zPattern); if( c>=prior_c && c<=c2 ) seen = 1; prior_c = 0; }else if( c==c2 ){ @@ -506,14 +510,14 @@ static int patternCompare( }else{ prior_c = c2; } - sqliteNextChar(zPattern); + SQLITE_SKIP_UTF8(zPattern); } if( c2==0 || (seen ^ invert)==0 ) return 0; - sqliteNextChar(zString); + SQLITE_SKIP_UTF8(zString); zPattern++; }else if( esc && !prevEscape && sqlite3ReadUtf8(zPattern)==esc){ prevEscape = 1; - sqliteNextChar(zPattern); + SQLITE_SKIP_UTF8(zPattern); }else{ if( noCase ){ if( sqlite3UpperToLower[c] != sqlite3UpperToLower[*zString] ) return 0; @@ -851,7 +855,7 @@ static void trimFunc( }else{ const unsigned char *z; for(z=zCharSet, nChar=0; *z; nChar++){ - sqliteNextChar(z); + SQLITE_SKIP_UTF8(z); } if( nChar>0 ){ azChar = sqlite3_malloc( nChar*(sizeof(char*)+1) ); @@ -861,7 +865,7 @@ static void trimFunc( aLen = (unsigned char*)&azChar[nChar]; for(z=zCharSet, nChar=0; *z; nChar++){ azChar[nChar] = z; - sqliteNextChar(z); + SQLITE_SKIP_UTF8(z); aLen[nChar] = z - azChar[nChar]; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a550ca8ab..70d26ed1b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.566 2007/05/12 12:08:51 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.567 2007/05/15 11:55:09 drh Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -1537,6 +1537,63 @@ typedef struct { extern int sqlite3_always_code_trigger_setup; /* +** A lookup table used by the SQLITE_READ_UTF8 macro. The definition +** is in utf.c. +*/ +extern const unsigned char sqlite3UtfTrans1[]; + +/* +** Macros for reading UTF8 characters. +** +** SQLITE_READ_UTF8(x,c) reads a single UTF8 value out of x and writes +** that value into c. The type of x must be unsigned char*. The type +** of c must be unsigned int. +** +** SQLITE_SKIP_UTF8(x) advances x forward by one character. The type of +** x must be unsigned char*. +** +** Notes On Invalid UTF-8: +** +** * These macros never allow a 7-bit character (0x00 through 0x7f) to +** be encoded as a multi-byte character. Any multi-byte character that +** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd. +** +** * These macros never allow a UTF16 surragate value to be encoded. +** If a multi-byte character attempts to encode a value between +** 0xd800 and 0xe000 then it is rendered as 0xfffd. +** +** * Bytes in the range of 0x80 through 0xbf which occur as the first +** byte of a character are interpreted as single-byte characters +** and rendered as themselves even though they are technically +** invalid characters. +** +** * These routines accept an infinite number of different UTF8 encodings +** for unicode values 0x80 and greater. They do not change over-length +** encodings to 0xfffd as some systems recommend. +** +*/ +#define SQLITE_READ_UTF8(zIn, c) { \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3UtfTrans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } \ +} +#define SQLITE_SKIP_UTF8(zIn) { \ + if( (*(zIn++))>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + + + + +/* ** The SQLITE_CORRUPT_BKPT macro can be either a constant (for production ** builds) or a function call (for debugging). If it is a function call, ** it allows the operator to set a breakpoint at the spot where database @@ -1753,7 +1810,7 @@ int sqlite3GetInt32(const char *, int*); int sqlite3FitsIn64Bits(const char *); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); -int sqlite3ReadUtf8(const unsigned char *); +u32 sqlite3ReadUtf8(const unsigned char *); int sqlite3PutVarint(unsigned char *, u64); int sqlite3GetVarint(const unsigned char *, u64 *); int sqlite3GetVarint32(const unsigned char *, u32 *); @@ -12,7 +12,7 @@ ** This file contains routines used to translate between UTF-8, ** UTF-16, UTF-16BE, and UTF-16LE. ** -** $Id: utf.c,v 1.46 2007/05/10 17:23:12 drh Exp $ +** $Id: utf.c,v 1.47 2007/05/15 11:55:09 drh Exp $ ** ** Notes on UTF-8: ** @@ -34,29 +34,6 @@ ** 0xff 0xfe little-endian utf-16 follows ** 0xfe 0xff big-endian utf-16 follows ** -** -** Handling of malformed strings: -** -** SQLite accepts and processes malformed strings without an error wherever -** possible. However this is not possible when converting between UTF-8 and -** UTF-16. -** -** When converting malformed UTF-8 strings to UTF-16, one instance of the -** replacement character U+FFFD for each byte that cannot be interpeted as -** part of a valid unicode character. -** -** When converting malformed UTF-16 strings to UTF-8, one instance of the -** replacement character U+FFFD for each pair of bytes that cannot be -** interpeted as part of a valid unicode character. -** -** This file contains the following public routines: -** -** sqlite3VdbeMemTranslate() - Translate the encoding used by a Mem* string. -** sqlite3VdbeMemHandleBom() - Handle byte-order-marks in UTF16 Mem* strings. -** sqlite3Utf16ByteLen() - Calculate byte-length of a void* UTF16 string. -** sqlite3Utf8CharLen() - Calculate char-length of a char* UTF8 string. -** sqlite3Utf8LikeCompare() - Do a LIKE match given two UTF8 char* strings. -** */ #include "sqliteInt.h" #include <assert.h> @@ -69,89 +46,20 @@ const int sqlite3one = 1; /* -** This table maps from the first byte of a UTF-8 character to the number -** of trailing bytes expected. A value '4' indicates that the table key -** is not a legal first byte for a UTF-8 character. -*/ -static const u8 xtra_utf8_bytes[256] = { -/* 0xxxxxxx */ -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - -/* 10wwwwww */ -4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - -/* 110yyyyy */ -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - -/* 1110zzzz */ -2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - -/* 11110yyy */ -3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, -}; - -/* -** This table maps from the number of trailing bytes in a UTF-8 character -** to an integer constant that is effectively calculated for each character -** read by a naive implementation of a UTF-8 character reader. The code -** in the READ_UTF8 macro explains things best. +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. */ -static const int xtra_utf8_bits[] = { - 0, - 12416, /* (0xC0 << 6) + (0x80) */ - 925824, /* (0xE0 << 12) + (0x80 << 6) + (0x80) */ - 63447168 /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ +const unsigned char sqlite3UtfTrans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, }; -/* -** If a UTF-8 character contains N bytes extra bytes (N bytes follow -** the initial byte so that the total character length is N+1) then -** masking the character with utf8_mask[N] must produce a non-zero -** result. Otherwise, we have an (illegal) overlong encoding. -*/ -static const int utf_mask[] = { - 0x00000000, - 0xffffff80, - 0xfffff800, - 0xffff0000, -}; - -#define READ_UTF8(zIn, c) { \ - int xtra; \ - c = *(zIn)++; \ - xtra = xtra_utf8_bytes[c]; \ - switch( xtra ){ \ - case 4: c = (int)0xFFFD; break; \ - case 3: c = (c<<6) + *(zIn)++; \ - case 2: c = (c<<6) + *(zIn)++; \ - case 1: c = (c<<6) + *(zIn)++; \ - c -= xtra_utf8_bits[xtra]; \ - if( (utf_mask[xtra]&c)==0 \ - || (c&0xFFFFF800)==0xD800 \ - || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ - } \ -} -int sqlite3ReadUtf8(const unsigned char *z){ - int c; - READ_UTF8(z, c); - return c; -} - -#define SKIP_UTF8(zIn) { \ - zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1); \ -} - #define WRITE_UTF8(zOut, c) { \ if( c<0x00080 ){ \ *zOut++ = (c&0xFF); \ @@ -346,14 +254,14 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ if( desiredEnc==SQLITE_UTF16LE ){ /* UTF-8 -> UTF-16 Little-endian */ while( zIn<zTerm ){ - READ_UTF8(zIn, c); + SQLITE_READ_UTF8(zIn, c); WRITE_UTF16LE(z, c); } }else{ assert( desiredEnc==SQLITE_UTF16BE ); /* UTF-8 -> UTF-16 Big-endian */ while( zIn<zTerm ){ - READ_UTF8(zIn, c); + SQLITE_READ_UTF8(zIn, c); WRITE_UTF16BE(z, c); } } @@ -457,17 +365,18 @@ int sqlite3VdbeMemHandleBom(Mem *pMem){ ** number of unicode characters in the first nByte of pZ (or up to ** the first 0x00, whichever comes first). */ -int sqlite3Utf8CharLen(const char *z, int nByte){ +int sqlite3Utf8CharLen(const char *zIn, int nByte){ int r = 0; - const char *zTerm; + const u8 *z = (const u8*)zIn; + const u8 *zTerm; if( nByte>=0 ){ zTerm = &z[nByte]; }else{ - zTerm = (const char *)(-1); + zTerm = (const u8*)(-1); } assert( z<=zTerm ); while( *z!=0 && z<zTerm ){ - SKIP_UTF8(z); + SQLITE_SKIP_UTF8(z); r++; } return r; @@ -589,7 +498,7 @@ int sqlite3Utf8To8(unsigned char *zIn){ int c; while(1){ - READ_UTF8(zIn, c); + SQLITE_READ_UTF8(zIn, c); if( c==0 ) break; if( c!=0xfffd ){ WRITE_UTF8(zOut, c); @@ -617,8 +526,9 @@ void sqlite3UtfSelfTest(){ z = zBuf; WRITE_UTF8(z, i); n = z-zBuf; + z[0] = 0; z = zBuf; - READ_UTF8(z, c); + SQLITE_READ_UTF8(z, c); t = i; if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD; if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD; @@ -630,6 +540,7 @@ void sqlite3UtfSelfTest(){ z = zBuf; WRITE_UTF16LE(z, i); n = z-zBuf; + z[0] = 0; z = zBuf; READ_UTF16LE(z, c); assert( c==i ); @@ -640,6 +551,7 @@ void sqlite3UtfSelfTest(){ z = zBuf; WRITE_UTF16BE(z, i); n = z-zBuf; + z[0] = 0; z = zBuf; READ_UTF16BE(z, c); assert( c==i ); |