diff options
Diffstat (limited to 'src/json.c')
| -rw-r--r-- | src/json.c | 2880 |
1 files changed, 2580 insertions, 300 deletions
diff --git a/src/json.c b/src/json.c index 0f11ace02..def5e63d3 100644 --- a/src/json.c +++ b/src/json.c @@ -15,19 +15,128 @@ ** This file began as an extension in ext/misc/json1.c in 2015. That ** extension proved so useful that it has now been moved into the core. ** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.44.0 (pending), these routines also accept +** BLOB values that have JSON encoded using a binary representation we +** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk +** format SQLite JSONB is completely different and incompatible with +** PostgreSQL JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, @@ -74,11 +183,12 @@ static const char jsonIsOk[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - +/* Put code used only for testing inside the JSON_VVA() macro. +*/ #if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define VVA(X) +# define JSON_VVA(X) #else -# define VVA(X) X +# define JSON_VVA(X) X #endif /* Objects */ @@ -97,10 +207,15 @@ struct JsonString { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ + u8 eErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; +/* Allowed values for JsonString.eErr */ +#define JSTRING_OOM 0x01 /* Out of memory */ +#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ +#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ + /* A deferred cleanup task. A list of JsonCleanup objects might be ** run when the JsonParse object is destroyed. */ @@ -110,7 +225,7 @@ struct JsonCleanup { void *pArg; /* Argument to xOp() */ }; -/* JSON type values +/* JSON type values for JsonNode.eType */ #define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ #define JSON_NULL 1 @@ -122,17 +237,18 @@ struct JsonCleanup { #define JSON_ARRAY 7 #define JSON_OBJECT 8 -/* The "subtype" set for JSON values */ -#define JSON_SUBTYPE 74 /* Ascii for "J" */ - -/* -** Names of the various JSON types: +/* Human-readalbe names for the JsonNode types: */ static const char * const jsonType[] = { "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* Bit values for the JsonNode.jnFlag field */ #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ @@ -143,6 +259,16 @@ static const char * const jsonType[] = { #define JNODE_LABEL 0x20 /* Is a label of an object */ #define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ +/* +** Bit values for the flags passed into jsonExtractFunc() or +** jsonSetFunc() via the user-data value. +*/ +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ + /* A single node of parsed JSON. An array of these nodes describes ** a parse of JSON + edits. @@ -204,14 +330,30 @@ struct JsonParse { u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ + u8 isBinary; /* True if zJson is the binary encoding */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ u32 iHold; /* Age of this entry in the cache for LRU replacement */ + /* Storage for the binary JSONB format */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ + u8 *aBlob; /* BLOB representation of zJson */ + /* Search and edit information. See jsonLookupBlobStep() */ + u8 eEdit; /* Edit operation to apply */ + int delta; /* Size change due to the edit */ + u32 nIns; /* Number of bytes to insert */ + u8 *aIns; /* Content to be inserted */ }; +/* Allowed values for JsonParse.eEdit */ +#define JEDIT_DEL 0x01 /* Delete if exists */ +#define JEDIT_INS 0x02 /* Insert if not exists */ +#define JEDIT_REPL 0x04 /* Overwrite if exists */ +#define JEDIT_SET 0x08 /* Insert or overwrite */ + /* ** Maximum nesting depth of JSON for this implementation. ** @@ -222,12 +364,24 @@ struct JsonParse { #define JSON_MAX_DEPTH 1000 /************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static void jsonXlateNodeToBlob(JsonParse*,JsonNode*,JsonParse*); +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); +static int jsonXlateBlobToNode(JsonParse *pParse, u32 i); +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); +static u32 jsonXlateBlobToText(JsonParse*,u32,JsonString*); + +/************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ -/* Set the JsonString object to an empty string +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. */ -static void jsonZero(JsonString *p){ +static void jsonStringZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; @@ -236,39 +390,39 @@ static void jsonZero(JsonString *p){ /* Initialize the JsonString object */ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->bErr = 0; - jsonZero(p); + p->eErr = 0; + jsonStringZero(p); } /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(JsonString *p){ +static void jsonStringReset(JsonString *p){ if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); - jsonZero(p); + jsonStringZero(p); } /* Report an out-of-memory (OOM) condition */ -static void jsonOom(JsonString *p){ - p->bErr = 1; +static void jsonStringOom(JsonString *p){ + p->eErr |= JSTRING_OOM; sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); + jsonStringReset(p); } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(JsonString *p, u32 N){ +static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->bErr ) return 1; + if( p->eErr ) return 1; zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ - jsonOom(p); + jsonStringOom(p); return SQLITE_NOMEM; } memcpy(zNew, p->zBuf, (size_t)p->nUsed); @@ -277,8 +431,8 @@ static int jsonGrow(JsonString *p, u32 N){ }else{ p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); if( p->zBuf==0 ){ - p->bErr = 1; - jsonZero(p); + p->eErr |= JSTRING_OOM; + jsonStringZero(p); return SQLITE_NOMEM; } } @@ -288,29 +442,29 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static SQLITE_NOINLINE void jsonAppendExpand( +static SQLITE_NOINLINE void jsonStringExpandAndAppend( JsonString *p, const char *zIn, u32 N ){ assert( N>0 ); - if( jsonGrow(p,N) ) return; + if( jsonStringGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } } static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ - assert( N>0 ); + if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -322,7 +476,7 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; va_start(ap, zFormat); sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); va_end(ap); @@ -332,7 +486,7 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ - if( jsonGrow(p,1) ) return; + if( jsonStringGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } static void jsonAppendChar(JsonString *p, char c){ @@ -343,6 +497,14 @@ static void jsonAppendChar(JsonString *p, char c){ } } +/* Make sure there is a zero terminator on p->zBuf[] +*/ +static void jsonStringTerminate(JsonString *p){ + if( p->nUsed<p->nAlloc || jsonStringGrow(p,1) ){ + p->zBuf[p->nUsed] = 0; + } +} + /* Try to force the string to be a zero-terminated RCStr string. ** ** Return true on success. Return false if an OOM prevents this @@ -350,12 +512,12 @@ static void jsonAppendChar(JsonString *p, char c){ */ static int jsonForceRCStr(JsonString *p){ jsonAppendChar(p, 0); - if( p->bErr ) return 0; + if( p->eErr ) return 0; p->nUsed--; if( p->bStatic==0 ) return 1; p->nAlloc = 0; p->nUsed++; - jsonGrow(p, p->nUsed); + jsonStringGrow(p, p->nUsed); p->nUsed--; return p->bStatic==0; } @@ -379,7 +541,8 @@ static void jsonAppendSeparator(JsonString *p){ */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 i; - if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; + if( zIn==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; for(i=0; i<N; i++){ unsigned char c = ((unsigned const char*)zIn)[i]; @@ -387,7 +550,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = c; }else if( c=='"' || c=='\\' ){ json_simple_escape: - if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + if( (p->nUsed+N+3-i > p->nAlloc) && jsonStringGrow(p,N+3-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ @@ -408,7 +571,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ c = aSpecial[c]; goto json_simple_escape; } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + if( (p->nUsed+N+7+i > p->nAlloc) && jsonStringGrow(p,N+7-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = 'u'; p->zBuf[p->nUsed++] = '0'; @@ -429,15 +592,17 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ u32 i; jsonAppendChar(p, '"'); - zIn++; - N -= 2; while( N>0 ){ for(i=0; i<N && zIn[i]!='\\' && zIn[i]!='"'; i++){} if( i>0 ){ jsonAppendRawNZ(p, zIn, i); + if( i>=N ) break; zIn += i; N -= i; - if( N==0 ) break; + } + if( N<2 ){ + p->eErr |= JSTRING_MALFORMED; + break; } if( zIn[0]=='"' ){ jsonAppendRawNZ(p, "\\\"", 2); @@ -454,6 +619,11 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, "\\u0009", 6); break; case 'x': + if( N<4 ){ + N = 2; + p->eErr |= JSTRING_MALFORMED; + break; + } jsonAppendRawNZ(p, "\\u00", 4); jsonAppendRawNZ(p, &zIn[2], 2); zIn += 2; @@ -463,14 +633,22 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, "\\u0000", 6); break; case '\r': - if( zIn[2]=='\n' ){ + if( N>2 && zIn[2]=='\n' ){ zIn++; N--; } break; case '\n': break; - case 0xe2: + case 0xe2: /* \ followed by U+2028 or U+2029 line terminator ignored */ + if( N<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + N = 2; + p->eErr |= JSTRING_MALFORMED; + break; + } assert( N>=4 ); assert( 0x80==(u8)zIn[2] ); assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); @@ -481,6 +659,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, zIn, 2); break; } + assert( N>=2 ); zIn += 2; N -= 2; } @@ -493,6 +672,14 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + char *zBuf = sqlite3_malloc64( N+1 ); + if( zBuf==0 ){ + p->eErr |= JSTRING_OOM; + return; + } + memcpy(zBuf, zIn, N); + zBuf[N] = 0; + zIn = zBuf; if( zIn[0]=='+' ){ zIn++; N--; @@ -510,10 +697,11 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ assert( rc==2 ); jsonAppendRawNZ(p, "9.0e999", 7); } - return; + }else{ + assert( N>0 ); + jsonAppendRawNZ(p, zIn, N); } - assert( N>0 ); - jsonAppendRawNZ(p, zIn, N); + sqlite3_free(zBuf); } /* @@ -549,13 +737,11 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ } } - - /* -** Append a function parameter value to the JSON string under -** construction. +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. */ -static void jsonAppendValue( +static void jsonAppendSqlValue( JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ @@ -585,10 +771,16 @@ static void jsonAppendValue( break; } default: { - if( p->bErr==0 ){ + if( jsonFuncArgMightBeBinary(pValue) ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(pValue); + px.nBlob = sqlite3_value_bytes(pValue); + jsonXlateBlobToText(&px, 0, p); + }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; - jsonReset(p); + p->eErr = JSTRING_ERR; + jsonStringReset(p); } break; } @@ -596,13 +788,17 @@ static void jsonAppendValue( } -/* Make the JSON in p the result of the SQL function. +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. ** -** The JSON string is reset. +** The JsonString is reset. */ -static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - if( p->bStatic ){ +static void jsonReturnString(JsonString *p){ + if( p->eErr==0 ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ @@ -610,12 +806,15 @@ static void jsonResult(JsonString *p){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); + }else{ + sqlite3_result_error_nomem(p->pCtx); } - } - if( p->bErr==1 ){ + }else if( p->eErr & JSTRING_OOM ){ sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_MALFORMED ){ + sqlite3_result_error(p->pCtx, "malformed JSON", -1); } - jsonReset(p); + jsonStringReset(p); } /************************************************************************** @@ -710,11 +909,10 @@ static int jsonParseAddCleanup( } /* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. +** Translate the JsonNode pNode into a pure JSON string and +** append that string on pOut. Subsubstructure is also included. */ -static void jsonRenderNode( +static void jsonXlateNodeToText( JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ JsonString *pOut /* Write JSON here */ @@ -752,18 +950,13 @@ static void jsonRenderNode( case JSON_STRING: { assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->jnFlags & JNODE_LABEL ){ - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - jsonAppendChar(pOut, '"'); - }else{ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - } + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ - assert( pNode->n>0 ); + jsonAppendChar(pOut, '"'); jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); } break; } @@ -794,7 +987,7 @@ static void jsonRenderNode( while( j<=pNode->n ){ if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonXlateNodeToText(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } @@ -811,12 +1004,12 @@ static void jsonRenderNode( u32 j = 1; jsonAppendChar(pOut, '{'); for(;;){ - while( j<=pNode->n ){ + while( j<pNode->n ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonXlateNodeToText(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(pParse, &pNode[j+1], pOut); + jsonXlateNodeToText(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -833,28 +1026,48 @@ static void jsonRenderNode( } /* -** Return a JsonNode and all its descendants as a JSON string. +** Make the return value of an SQL function be the JSON encoded by pNode. +** +** By default, the node is rendered as RFC-8259 JSON text (canonical +** JSON text without any JSON-5 enhancements). However if the +** JSON_BLOB flag is set in the user-data for the function, then the +** node is rendered into the JSONB format and returned as a BLOB. */ -static void jsonReturnJson( +static void jsonReturnNodeAsJson( JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ int bGenerateAlt, /* Also store the rendered text in zAlt */ int omitSubtype /* Do not call sqlite3_result_subtype() */ ){ + int flags; JsonString s; if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); return; } - if( pParse->nErr==0 ){ - jsonInit(&s, pCtx); - jsonRenderNode(pParse, pNode, &s); + if( pParse->nErr ){ + return; + } + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + JsonParse x; + memset(&x, 0, sizeof(x)); + jsonXlateNodeToBlob(pParse, pNode, &x); + if( x.oom ){ + sqlite3_result_error_nomem(pCtx); + sqlite3_free(x.aBlob); + }else{ + sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); + } + }else{ + jsonStringInit(&s, pCtx); + jsonXlateNodeToText(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ pParse->zAlt = sqlite3RCStrRef(s.zBuf); pParse->nAlt = s.nUsed; } - jsonResult(&s); + jsonReturnString(&s); if( !omitSubtype ) sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } } @@ -865,7 +1078,7 @@ static void jsonReturnJson( ** character: 0..9a..fA..F */ static u8 jsonHexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); + if( !sqlite3Isxdigit(h) ) return 0; #ifdef SQLITE_EBCDIC h += 9*(1&~(h>>4)); #else @@ -879,10 +1092,6 @@ static u8 jsonHexToInt(int h){ */ static u32 jsonHexToInt4(const char *z){ u32 v; - assert( sqlite3Isxdigit(z[0]) ); - assert( sqlite3Isxdigit(z[1]) ); - assert( sqlite3Isxdigit(z[2]) ); - assert( sqlite3Isxdigit(z[3]) ); v = (jsonHexToInt(z[0])<<12) + (jsonHexToInt(z[1])<<8) + (jsonHexToInt(z[2])<<4) @@ -891,9 +1100,16 @@ static u32 jsonHexToInt4(const char *z){ } /* -** Make the JsonNode the return value of the function. +** Make the return value from an SQL function be the SQL value of +** JsonNode pNode. +** +** If pNode is an atom (not an array or object) then the value returned +** is a pure SQL value - an SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT, or +** SQLITE_NULL. However, if pNode is a JSON array or object, then the +** value returned is either RFC-8259 JSON text or a BLOB in the JSONB +** format, depending on the JSON_BLOB flag of the function user-data. */ -static void jsonReturn( +static void jsonReturnFromNode( JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ @@ -918,12 +1134,20 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; + char *zz; + sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( pNode->eU==1 ); - z = pNode->u.zJContent; + zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); + if( zz==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + z = zz; if( z[0]=='-' ){ z++; bNeg = 1; } else if( z[0]=='+' ){ z++; } rc = sqlite3DecOrHexToI64(z, &i); + sqlite3DbFree(db, zz); if( rc<=1 ){ sqlite3_result_int64(pCtx, bNeg ? -i : i); }else if( rc==3 && bNeg ){ @@ -939,7 +1163,7 @@ static void jsonReturn( assert( pNode->eU==1 ); to_double: z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3AtoF(z, &r, pNode->n, SQLITE_UTF8); sqlite3_result_double(pCtx, r); break; } @@ -951,7 +1175,7 @@ static void jsonReturn( }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ @@ -968,7 +1192,7 @@ static void jsonReturn( sqlite3_result_error_nomem(pCtx); break; } - for(i=1, j=0; i<n-1; i++){ + for(i=0, j=0; i<n; i++){ char c = z[i]; if( c=='\\' ){ c = z[++i]; @@ -1043,15 +1267,12 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx, 0, omitSubtype); + jsonReturnNodeAsJson(pParse, pNode, pCtx, 0, omitSubtype); break; } } } -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - /* ** A macro to hint to the compiler that a function should not be ** inlined. @@ -1114,7 +1335,7 @@ static int jsonParseAddNode( assert( p!=0 ); p->eType = (u8)(eType & 0xff); p->jnFlags = (u8)(eType >> 8); - VVA( p->eU = zContent ? 1 : 0 ); + JSON_VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; return pParse->nNode++; @@ -1327,19 +1548,22 @@ static const struct NanInfName { }; /* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. +** Translate a single element of JSON text beginning at pParse->zJson[i] into +** its JsonNode representation. Append the translation onto the +** pParse->aNode[] array, which is increased in size as necessary. +** Return the pJson->zJson[] index of the first character past the end of +** the element that was parsed. ** ** Special return values: ** ** 0 End of input ** -1 Syntax error -** -2 '}' seen -** -3 ']' seen -** -4 ',' seen -** -5 ':' seen +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / */ -static int jsonParseValue(JsonParse *pParse, u32 i){ +static int jsonXlateTextToNode(JsonParse *pParse, u32 i){ char c; u32 j; int iThis; @@ -1358,7 +1582,7 @@ json_parse_restart: } for(j=i+1;;j++){ u32 nNode = pParse->nNode; - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; @@ -1375,7 +1599,7 @@ json_parse_restart: ){ k++; } - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); + jsonParseAddNode(pParse, JSON_STRING, k-j, &z[j]); pParse->hasNonstd = 1; x = k; }else{ @@ -1401,7 +1625,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1409,7 +1633,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1428,7 +1652,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1441,7 +1665,9 @@ json_parse_restart: pParse->iErr = j; return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } pParse->iDepth--; return j+1; } @@ -1455,7 +1681,7 @@ json_parse_restart: } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1479,7 +1705,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1538,7 +1764,7 @@ json_parse_restart: return -1; } } - jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); + jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j-1-i, &z[i+1]); return j+1; } case 't': { @@ -1656,7 +1882,10 @@ json_parse_restart: } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') + && ALWAYS(j-2>=i) + && sqlite3Isdigit(z[j-2]) + ){ pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; }else{ @@ -1770,13 +1999,42 @@ json_parse_restart: } /* End switch(z[i]) */ } +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + /* -** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory held by pParse, -** but not pParse itself. +** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB +** blob) into the JsonNode representation. ** -** pParse must be initialized to an empty parse object prior to calling -** this routine. +** Return 0 on success or non-zero if there are any errors. +** If an error occurs, free all memory held by pParse, but not pParse itself. +** +** pParse must be initialized with pParse->zJson set to the input text or +** blob prior to calling this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ @@ -1784,9 +2042,15 @@ static int jsonParse( ){ int i; const char *zJson = pParse->zJson; - i = jsonParseValue(pParse, 0); + if( pParse->isBinary ){ + pParse->aBlob = (u8*)pParse->zJson; + pParse->nBlob = pParse->nJson; + i = jsonXlateBlobToNode(pParse, 0); + }else{ + i = jsonXlateTextToNode(pParse, 0); + } if( pParse->oom ) i = -1; - if( i>0 ){ + if( !pParse->isBinary && i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; if( zJson[i] ){ @@ -1812,34 +2076,6 @@ static int jsonParse( return 0; } - -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - /* ** Compute the parentage of all nodes in a completed parse. */ @@ -1895,8 +2131,8 @@ static JsonParse *jsonParseCached( sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ int bUnedited /* No prior edits allowed */ ){ - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); + char *zJson; + int nJson; JsonParse *p; JsonParse *pMatch = 0; int iKey; @@ -1904,6 +2140,16 @@ static JsonParse *jsonParseCached( u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; int bJsonRCStr; + int isBinary; + + if( jsonFuncArgMightBeBinary(pJson) ){ + zJson = (char*)sqlite3_value_blob(pJson); + isBinary = 1; + }else{ + zJson = (char*)sqlite3_value_text(pJson); + isBinary = 0; + } + nJson = sqlite3_value_bytes(pJson); if( zJson==0 ) return 0; for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){ @@ -1962,9 +2208,11 @@ static JsonParse *jsonParseCached( p->bJsonIsRCStr = 1; }else{ p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+1); + memcpy(p->zJson, zJson, nJson+(isBinary==0)); } p->nJPRef = 1; + p->isBinary = isBinary; + p->nJson = nJson; if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; @@ -1974,7 +2222,6 @@ static JsonParse *jsonParseCached( jsonParseFree(p); return 0; } - p->nJson = nJson; p->iHold = iMaxHold+1; /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, @@ -1987,14 +2234,9 @@ static JsonParse *jsonParseCached( ** a match. */ static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } + if( pNode->eType!=JSON_STRING ) return 0; + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; } static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ if( p1->jnFlags & JNODE_RAW ){ @@ -2006,9 +2248,6 @@ static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ } } -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such @@ -2105,7 +2344,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; } return pNode; @@ -2124,7 +2363,9 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + i++; + } j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; @@ -2187,7 +2428,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); } return pNode; } @@ -2224,11 +2465,21 @@ static JsonNode *jsonLookupAppend( } /* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). +** Compute the text of an error in JSON path syntax. +** +** If ctx is not NULL then push the error message into ctx and return NULL. +* If ctx is NULL, then return the text of the error message. */ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); +static char *jsonPathSyntaxError(const char *zErr, sqlite3_context *ctx){ + char *zMsg = sqlite3_mprintf("JSON path error near '%q'", zErr); + if( ctx==0 ) return zMsg; + if( zMsg==0 ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + } + return 0; } /* @@ -2249,7 +2500,6 @@ static JsonNode *jsonLookup( ){ const char *zErr = 0; JsonNode *pNode = 0; - char *zMsg; if( zPath==0 ) return 0; if( zPath[0]!='$' ){ @@ -2263,13 +2513,7 @@ static JsonNode *jsonLookup( lookup_err: pParse->nErr++; assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); - if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); - }else{ - sqlite3_result_error_nomem(pCtx); - } + jsonPathSyntaxError(zErr, pCtx); return 0; } @@ -2307,6 +2551,1895 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ } } +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + + +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u32 t; + assert( N>pParse->nBlobAlloc ); + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; + } + if( t<N ) t = N+100; + aNew = sqlite3_realloc64( pParse->aBlob, t ); + if( aNew==0 ){ pParse->oom = 1; return 1; } + pParse->aBlob = aNew; + pParse->nBlobAlloc = t; + return 0; +} + +/* +** If pParse->aBlob is not previously editable (because it is taken +** from sqlite3_value_blob(), as indicated by the fact that +** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable +** by making a copy into space obtained from malloc. +** +** Return true on success. Return false on OOM. +*/ +static int jsonBlobMakeEditable(JsonParse *pParse){ + u8 *aOld; + u32 nSize; + if( pParse->nBlobAlloc>0 ) return 1; + aOld = pParse->aBlob; + nSize = pParse->nBlob + pParse->nIns; + if( nSize>100 ) nSize -= 100; + pParse->aBlob = 0; + if( jsonBlobExpand(pParse, nSize) ){ + return 0; + } + assert( pParse->nBlobAlloc >= pParse->nBlob + pParse->nIns ); + memcpy(pParse->aBlob, aOld, pParse->nBlob); + return 1; +} + + +/* Expand pParse->aBlob and append N bytes. +** +** Return the number of errors. +*/ +static SQLITE_NOINLINE int jsonBlobExpandAndAppend( + JsonParse *pParse, + const u8 *aData, + u32 N +){ + if( jsonBlobExpand(pParse, pParse->nBlob+N) ) return 1; + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append a single character. Return 1 if an error occurs. +*/ +static int jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, &c, 1); + } + pParse->aBlob[pParse->nBlob++] = c; + return 0; +} + +/* Append bytes. Return 1 if an error occurs. +*/ +static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ + if( pParse->nBlob+N > pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, aData, N); + } + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append an node type byte together with the payload size. +*/ +static void jsonBlobAppendNodeType( + JsonParse *pParse, + u8 eType, + u32 szPayload +){ + u8 a[5]; + if( szPayload<=11 ){ + jsonBlobAppendOneByte(pParse, eType | (szPayload<<4)); + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 2); + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 3); + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 5); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static void jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a; + u8 szType; + u8 nExtra; + u8 nNeeded; + i8 delta; + if( pParse->oom ) return; + a = &pParse->aBlob[i]; + szType = a[0]>>4; + if( szType<=11 ){ + nExtra = 0; + }else if( szType==12 ){ + nExtra = 1; + }else if( szType==13 ){ + nExtra = 2; + }else{ + nExtra = 4; + } + if( szPayload<=11 ){ + nNeeded = 0; + }else if( szPayload<=0xff ){ + nNeeded = 1; + }else if( szPayload<=0xffff ){ + nNeeded = 2; + }else{ + nNeeded = 4; + } + delta = nNeeded - nExtra; + if( delta ){ + u32 newSize = pParse->nBlob + delta; + if( delta>0 ){ + if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ + return; /* OOM error. Error state recorded in pParse->oom. */ + } + a = &pParse->aBlob[i]; + memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); + }else{ + memmove(&a[1], &a[1-delta], pParse->nBlob - (i+1-delta)); + } + pParse->nBlob = newSize; + } + if( nNeeded==0 ){ + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( nNeeded==1 ){ + a[0] = (a[0] & 0x0f) | 0xc0; + a[1] = szPayload & 0xff; + }else if( nNeeded==2 ){ + a[0] = (a[0] & 0x0f) | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + a[0] = (a[0] & 0x0f) | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } +} + +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !sqlite3Isxdigit(z[1]) ) return 0; + if( !sqlite3Isxdigit(z[2]) ) return 0; + if( !sqlite3Isxdigit(z[3]) ) return 0; + if( !sqlite3Isxdigit(z[4]) ) return 0; + *pOp = JSONB_TEXTJ; + return 1; +} + +/* +** Translate a single element of JSON text at pParse->zJson[i] into +** its equivalent binary JSONB representation. Append the translation into +** pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. +** +** Return the index of the first character past the end of the element parsed, +** or one of the following special result codes: +** +** 0 End of input +** -1 Syntax error +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / +*/ +static int jsonXlateTextToBlob(JsonParse *pParse, u32 i){ + char c; + u32 j; + u32 iThis, iStart; + int x; + u8 t; + const char *z = pParse->zJson; +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { + /* Parse object */ + iThis = pParse->nBlob; + jsonBlobAppendNodeType(pParse, JSONB_OBJECT, (pParse->nJson-i)*2); + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + iStart = pParse->nBlob; + for(j=i+1;;j++){ + u32 iBlob = pParse->nBlob; + x = jsonXlateTextToBlob(pParse, j); + if( x<=0 ){ + int op; + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) + ){ + k++; + } + assert( iBlob==pParse->nBlob ); + jsonBlobAppendNodeType(pParse, op, k-j); + jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } + } + if( pParse->oom ) return -1; + t = pParse->aBlob[iBlob] & 0x0f; + if( t<JSONB_TEXT || t>JSONB_TEXTRAW ){ + pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==':' ){ + j++; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonXlateTextToBlob(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: + x = jsonXlateTextToBlob(pParse, j); + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonXlateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; + return j+1; + } + case '[': { + /* Parse array */ + iThis = pParse->nBlob; + jsonBlobAppendNodeType(pParse, JSONB_ARRAY, pParse->nJson - i); + iStart = pParse->nBlob; + if( pParse->oom ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + for(j=i+1;;j++){ + x = jsonXlateTextToBlob(pParse, j); + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonXlateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; + return j+1; + } + case '\'': { + u8 opcode; + char cDelim; + int nn; + pParse->hasNonstd = 1; + opcode = JSONB_TEXT; + goto parse_string; + case '"': + /* Parse string */ + opcode = JSONB_TEXT; + parse_string: + cDelim = z[i]; + nn = pParse->nJson; + for(j=i+1; j<nn; j++){ + if( jsonIsOk[(unsigned char)z[j]] ) continue; + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else{ + pParse->iErr = j; + return -1; + } + }else if( c<=0x1f ){ + /* Control characters are not allowed in strings */ + pParse->iErr = j; + return -1; + } + } + jsonBlobAppendNodeType(pParse, opcode, j-1-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i+1], j-1-i); + return j+1; + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_TRUE); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonBlobAppendOneByte(pParse, JSONB_FALSE); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenE; + pParse->hasNonstd = 1; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + seenE = 0; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + /* Parse number */ + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + parse_number: + seenE = 0; + assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + + if( c<='0' ){ + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( t==0x00 ); + pParse->hasNonstd = 1; + t = 0x01; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 6); + jsonBlobAppendNBytes(pParse, (const u8*)"-9e999", 6); + }else{ + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 5); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + t |= 0x01; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } + } + + parse_number_2: + for(j=i+1;; j++){ + c = z[j]; + if( sqlite3Isdigit(c) ) continue; + if( c=='.' ){ + if( (t & 0x02)!=0 ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } + continue; + } + break; + } + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; + jsonBlobAppendNodeType(pParse, JSONB_INT+t, j-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); + return j; + } + case '}': { + pParse->iErr = i; + return -2; /* End of {...} */ + } + case ']': { + pParse->iErr = i; + return -3; /* End of [...] */ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { + return 0; /* End of file */ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; k<sizeof(aNanInfName)/sizeof(aNanInfName[0]); k++){ + if( c!=aNanInfName[k].c1 && c!=aNanInfName[k].c2 ) continue; + nn = aNanInfName[k].n; + if( sqlite3StrNICmp(&z[i], aNanInfName[k].zMatch, nn)!=0 ){ + continue; + } + if( sqlite3Isalnum(z[i+nn]) ) continue; + if( aNanInfName[k].eType==JSON_REAL ){ + jsonBlobAppendOneByte(pParse, JSONB_FLOAT | 0x50); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + }else{ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + } + pParse->hasNonstd = 1; + return i + nn; + } + pParse->iErr = i; + return -1; /* Syntax error */ + } + } /* End switch(z[i]) */ +} + + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. +** +** pParse must be initialized to an empty parse object prior to calling +** this routine. +*/ +static int jsonConvertTextToBlob( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx /* Report errors here */ +){ + int i; + const char *zJson = pParse->zJson; + i = jsonXlateTextToBlob(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( fast_isspace(zJson[i]) ) i++; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } + } + if( i<=0 ){ + if( ALWAYS(pCtx!=0) ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. +*/ +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + jsonStringTerminate(pStr); + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + (void)jsonXlateTextToBlob(&px, 0); + if( px.oom ){ + sqlite3_free(px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, sqlite3_free); + } +} + +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. Return 0 on error. +*/ +static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + if( NEVER(i>pParse->nBlob) ){ + *pSz = 0; + return 0; + } + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else if( x==13 ){ + if( i+2>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else{ + if( i+4>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + } + if( i+sz+n>pParse->nBlob ){ + sz = 0; + n = 0; + } + *pSz = sz; + return n; +} + + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonXlateBlobToText( + JsonParse *pParse, /* the complete parse of the JSON */ + u32 i, /* Start rendering at this index */ + JsonString *pOut /* Write JSON here */ +){ + u32 sz, n, j, iEnd; + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + jsonAppendRawNZ(pOut, "null", 4); + return i+1; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + return i+1; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + return i+1; + } + case JSONB_INT: + case JSONB_FLOAT: { + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + u32 k = 2; + sqlite3_uint64 u = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + } + for(; k<sz; k++){ + if( !sqlite3Isxdigit(zIn[k]) ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + }else{ + u = u*16 + sqlite3HexToInt(zIn[k]); + } + } + jsonPrintf(100,pOut,"%llu",u); + break; + } + case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ + u32 k = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + } + if( zIn[k]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(; k<sz; k++){ + jsonAppendChar(pOut, zIn[k]); + if( zIn[k]=='.' && (k+1==sz || !sqlite3Isdigit(zIn[k+1])) ){ + jsonAppendChar(pOut, '0'); + } + } + break; + } + case JSONB_TEXT: + case JSONB_TEXTJ: { + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + u32 sz2 = sz; + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz2>0 ){ + for(k=0; k<sz2 && zIn[k]!='\\'; k++){} + if( k>0 ){ + jsonAppendRawNZ(pOut, zIn, k); + if( k>=sz2 ){ + break; + } + zIn += k; + sz2 -= k; + } + if( sz2<2 ){ + if( sz2>0 ) pOut->eErr |= JSTRING_MALFORMED; + if( sz2==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u0009", 6); + break; + case 'x': + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 0; + break; + } + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz2 -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( sz2>2 && zIn[2]=='\n' ){ + zIn++; + sz2--; + } + break; + case '\n': + break; + case 0xe2: + /* '\' followed by either U+2028 or U+2029 is ignored as + ** whitespace. Not that in UTF8, U+2028 is 0xe2 0x80 0x29. + ** U+2029 is the same except for the last byte */ + if( sz2<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + pOut->eErr |= JSTRING_MALFORMED; + k = sz2; + break; + } + zIn += 2; + sz2 -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + if( sz2<2 ){ + sz2 = 0; + pOut->eErr |= JSTRING_MALFORMED; + break; + } + zIn += 2; + sz2 -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXTRAW: { + jsonAppendString(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + j = i+n; + iEnd = j+sz; + while( j<iEnd ){ + j = jsonXlateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, ','); + } + if( sz>0 ) pOut->nUsed--; + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + j = i+n; + iEnd = j+sz; + while( j<iEnd ){ + j = jsonXlateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + if( sz>0 ) pOut->nUsed--; + jsonAppendChar(pOut, '}'); + break; + } + + default: { + pOut->eErr |= JSTRING_MALFORMED; + break; + } + } + return i+n+sz; +} + +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + aBlob = sqlite3_value_blob(pJson); + if( aBlob==0 || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + if( sz+n!=(u32)nBlob ) return 0; + if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; + return sz+n==(u32)nBlob; +} + +/* Translate a single element of JSONB into the JsonNode format. The +** first byte of the element to be translated is at pParse->aBlob[i]. +** Return the index in pParse->aBlob[] of the first byte past the end +** of the JSONB element. Append the JsonNode translation in +** pParse->aNode[], which is increased in size as necessary. +*/ +static int jsonXlateBlobToNode(JsonParse *pParse, u32 i){ + u8 t; /* Node type */ + u32 sz; /* Node size */ + u32 x; /* Index of payload start */ + + const char *zPayload; + x = jsonbPayloadSize(pParse, i, &sz); + if( x==0 ) return -1; + t = pParse->zJson[i] & 0x0f; + zPayload = &pParse->zJson[i+x]; + switch( t ){ + case JSONB_NULL: { + if( sz>0 ) return -1; + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + break; + } + case JSONB_TRUE: { + if( sz>0 ) return -1; + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + break; + } + case JSONB_FALSE: { + if( sz>0 ) return -1; + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + break; + } + case JSONB_INT: { + if( sz==0 ) return -1; + jsonParseAddNode(pParse, JSON_INT, sz, zPayload); + break; + } + case JSONB_INT5: { + if( sz==0 ) return -1; + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_FLOAT: { + if( sz==0 ) return -1; + jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); + break; + } + case JSONB_FLOAT5: { + if( sz==0 ) return -1; + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_TEXTRAW: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXT: { + jsonParseAddNode(pParse, JSON_STRING, sz, zPayload); + break; + } + case JSONB_TEXTJ: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_ESCAPE<<8), sz, zPayload); + break; + } + case JSONB_TEXT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_STRING | ((JNODE_ESCAPE|JNODE_JSON5)<<8), + sz, zPayload); + break; + } + case JSONB_ARRAY: { + int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + u32 j = i+x; + while( j<i+x+sz ){ + int r = jsonXlateBlobToNode(pParse, j); + if( r<=0 ) return -1; + j = (u32)r; + } + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } + break; + } + case JSONB_OBJECT: { + int iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + u32 j = i+x, k = 0; + while( j<i+x+sz ){ + int r = jsonXlateBlobToNode(pParse, j); + if( r<=0 ) return -1; + if( (k++&1)==0 && !pParse->oom ){ + pParse->aNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; + } + j = (u32)r; + } + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } + if( k&1 ) return -1; + break; + } + default: { + return -1; + } + } + return i+x+sz; +} + +/* +** Translate pNode (which is always a node found in pParse->aNode[]) into +** the JSONB representation and append the translation onto the end of the +** pOut->aBlob[] array. +*/ +static void jsonXlateNodeToBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + JsonNode *pNode, /* The node to render */ + JsonParse *pOut /* Write the BLOB rendering of JSON here */ +){ + assert( pNode!=0 ); + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( i<pParse->nNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrev<i ); + if( pParse->aNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonBlobAppendNodeType(pOut, JSONB_NULL, 0); + break; + } + case JSON_TRUE: { + jsonBlobAppendNodeType(pOut, JSONB_TRUE, 0); + break; + } + case JSON_FALSE: { + jsonBlobAppendNodeType(pOut, JSONB_FALSE, 0); + break; + } + case JSON_STRING: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_RAW ){ + if( memchr(pNode->u.zJContent, '"', pNode->n)==0 + && memchr(pNode->u.zJContent, '\\', pNode->n)==0 + ){ + op = JSONB_TEXT; + }else{ + op = JSONB_TEXTRAW; + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_TEXT5; + }else{ + op = JSONB_TEXTJ; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_REAL: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_FLOAT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_FLOAT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_INT: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_INT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_INT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_ARRAY, pParse->nJson*2); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonXlateNodeToBlob(pParse, &pNode[j], pOut); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + case JSON_OBJECT: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_OBJECT, pParse->nJson*2); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonXlateNodeToBlob(pParse, &pNode[j], pOut); + jsonXlateNodeToBlob(pParse, &pNode[j+1], pOut); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + } +} + +/* +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. +*/ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; n>0 && i<iEnd; i+=sz+n, k++){ + n = jsonbPayloadSize(pParse, i, &sz); + } + return k; +} + +/* +** Edit the size of the element at iRoot by the amount in pParse->delta. +*/ +static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ + u32 sz = 0; + u32 nBlob; + assert( pParse->delta!=0 ); + assert( pParse->nBlobAlloc >= pParse->nBlob ); + nBlob = pParse->nBlob; + pParse->nBlob = pParse->nBlobAlloc; + (void)jsonbPayloadSize(pParse, iRoot, &sz); + pParse->nBlob = nBlob; + sz += pParse->delta; + jsonBlobChangePayloadSize(pParse, iRoot, sz); +} + +/* +** Error returns from jsonLookupBlobStep() +*/ +#define JSON_BLOB_ERROR 0xffffffff +#define JSON_BLOB_NOTFOUND 0xfffffffe +#define JSON_BLOB_PATHERROR 0xfffffffd +#define JSON_BLOB_ISERROR(x) ((x)>=JSON_BLOB_PATHERROR) + +/* +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. +** +** Return JSON_BLOB_NOTFOUND if no such element exists. +*/ +static u32 jsonLookupBlobStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ + const char *zPath, /* The path to search */ + u32 iLabel /* Label if iRoot is a value of in an object */ +){ + u32 i, j, k, nKey, sz, n, iEnd, rc; + const char *zKey; + u8 x; + + if( zPath[0]==0 ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse) ){ + n = jsonbPayloadSize(pParse, iRoot, &sz); + sz += n; + if( pParse->eEdit==JEDIT_DEL ){ + if( iLabel>0 ){ + sz += iRoot - iLabel; + iRoot = iLabel; + } + memmove(&pParse->aBlob[iRoot], &pParse->aBlob[iRoot+sz], + pParse->nBlob - (iRoot+sz)); + pParse->delta = -(int)sz; + pParse->nBlob -= sz; + }else if( pParse->eEdit==JEDIT_INS ){ + /* Already exists, so json_insert() is a no-op */ + }else{ + /* json_set() or json_replace() */ + int d = (int)pParse->nIns - (int)sz; + pParse->delta = d; + if( d!=0 ){ + if( pParse->nBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return iRoot; + } + memmove(&pParse->aBlob[iRoot+pParse->nIns], + &pParse->aBlob[iRoot+sz], + pParse->nBlob - iRoot - sz); + pParse->nBlob += d; + } + memcpy(&pParse->aBlob[iRoot], pParse->aIns, pParse->nIns); + } + } + return iRoot; + } + if( zPath[0]=='.' ){ + x = pParse->aBlob[iRoot]; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + return JSON_BLOB_PATHERROR; + } + testcase( nKey==0 ); + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + return JSON_BLOB_PATHERROR; + } + } + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; /* j is the index of a label */ + iEnd = j+sz; + while( j<iEnd ){ + x = pParse->aBlob[j] & 0x0f; + if( x<JSONB_TEXT || x>JSONB_TEXTRAW ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; + k = j+n; /* k is the index of the label text */ + if( k+sz>=iEnd ) return JSON_BLOB_ERROR; + if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ + u32 v = k+sz; /* v is the index of the value */ + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, v, &sz); + if( n==0 || v+n+sz>iEnd ) return JSON_BLOB_ERROR; + assert( j>0 ); + rc = jsonLookupBlobStep(pParse, v, &zPath[i], j); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + }else if( zPath[0]=='[' ){ + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; + } + if( i<2 || zPath[i]!=']' ){ + if( zPath[1]=='#' ){ + k = jsonbArrayCount(pParse, iRoot); + i = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int nn = 0; + i = 3; + do{ + nn = nn*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( nn>k ) return JSON_BLOB_NOTFOUND; + k -= nn; + } + if( zPath[i]!=']' ){ + return JSON_BLOB_PATHERROR; + } + }else{ + return JSON_BLOB_PATHERROR; + } + } + j = iRoot+n; + iEnd = j+sz; + while( j<iEnd ){ + if( k==0 ){ + rc = jsonLookupBlobStep(pParse, j, &zPath[i+1], 0); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + k--; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + if( k>1 ) return JSON_BLOB_NOTFOUND; + }else{ + return JSON_BLOB_PATHERROR; + } + if( pParse->eEdit==JEDIT_INS && jsonBlobMakeEditable(pParse) ){ + assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); + memmove(&pParse->aBlob[j], &pParse->aBlob[j+pParse->nIns], + pParse->nBlob - j); + memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); + pParse->delta = pParse->nIns; + pParse->nBlob += pParse->nIns; + jsonAfterEditSizeAdjust(pParse, iRoot); + return j; + } + return JSON_BLOB_NOTFOUND; +} + +/* +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. +*/ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob +){ + JsonParse x; + JsonString s; + + if( aBlob==0 ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&x, 0, &s); + jsonReturnString(&s); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx /* Return value for this function */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ) return; + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x = (char)pParse->aBlob[i+n]; + if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + if( bNeg ){ n--; sz++; } + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3_malloc( nOut+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(iIn=iOut=0; iIn<sz; iIn++){ + char c = z[iIn]; + if( c=='\\' ){ + c = z[++iIn]; + if( c=='u' ){ + u32 v = jsonHexToInt4(z+iIn+1); + iIn += 4; + if( v==0 ) break; + if( v<=0x7f ){ + zOut[iOut++] = (char)v; + }else if( v<=0x7ff ){ + zOut[iOut++] = (char)(0xc0 | (v>>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i<n-6 + && z[iIn+1]=='\\' + && z[iIn+2]=='u' + && ((vlo = jsonHexToInt4(z+iIn+3))&0xfc00)==0xdc00 + ){ + /* We have a surrogate pair */ + v = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + iIn += 6; + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); + iIn += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + iIn++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + iIn += 2; + continue; + }else{ + continue; + } + } /* end if( c=='\\' ) */ + zOut[iOut++] = c; + } /* end for() */ + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } + break; + } + default: { + sqlite3_result_error(pCtx, "malformed JSON", -1); + break; + } + } +} + +/* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. +*/ +static void jsonExtractFromBlob( + sqlite3_context *ctx, + sqlite3_value *pJson, + sqlite3_value *pPath, + int flags +){ + const char *zPath = (const char*)sqlite3_value_text(pPath); + u32 i = 0; + JsonParse px; + if( zPath==0 ) return; + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(pJson); + px.aBlob = (u8*)sqlite3_value_blob(pJson); + if( px.aBlob==0 ) return; + if( zPath[0]=='$' ){ + zPath++; + i = jsonLookupBlobStep(&px, 0, zPath, 0); + }else if( (flags & JSON_ABPATH) ){ + /* The -> and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + */ + JsonString jx; + jsonStringInit(&jx, ctx); + if( sqlite3Isdigit(zPath[0]) ){ + jsonAppendRawNZ(&jx, "[", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendRawNZ(&jx, "]", 2); + zPath = jx.zBuf; + }else if( zPath[0]!='[' ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendChar(&jx, 0); + zPath = jx.zBuf; + } + i = jsonLookupBlobStep(&px, 0, zPath, 0); + jsonStringReset(&jx); + }else{ + sqlite3_result_error(ctx, "bad path", -1); + return; + } + if( i<px.nBlob ){ + jsonReturnFromBlob(&px, i, ctx); + }else if( i==JSON_BLOB_NOTFOUND ){ + return; /* Return NULL if not found */ + }else if( i==JSON_BLOB_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + char *zMsg = sqlite3_mprintf("bad path syntax: %s", + sqlite3_value_text(pPath)); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + } +} + +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments are JSON paths of elements to be removed. Do that removal +** and return the result. +*/ +static void jsonRemoveFromBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + u32 rc; + const char *zPath = 0; + int flgs; + JsonParse px; + + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(argv[0]); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( px.aBlob==0 ) return; + for(i=1; i<argc; i++){ + const char *zPath = (const char*)sqlite3_value_text(argv[i]); + if( zPath==0 ) goto jsonRemoveFromBlob_patherror; + if( zPath[0]!='$' ) goto jsonRemoveFromBlob_patherror; + if( zPath[1]==0 ){ + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + return; /* return NULL if $ is removed */ + } + px.eEdit = JEDIT_DEL; + rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); + if( rc==JSON_BLOB_NOTFOUND ) continue; + if( JSON_BLOB_ISERROR(rc) ) goto jsonRemoveFromBlob_patherror; + } + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + sqlite3_result_blob(ctx, px.aBlob, px.nBlob, + px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&px, 0, &s); + jsonReturnString(&s); + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + } + return; + +jsonRemoveFromBlob_patherror: + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonPathSyntaxError(zPath, ctx); + return; +} + +/* +** pArg is a function argument that might be an SQL value or a JSON +** value. Figure out what it is and encode it as a JSONB blob. +** Return the results in pParse. +** +** pParse is uninitialized upon entry. This routine will handle the +** initialization of pParse. The result will be contained in +** pParse->aBlob and pParse->nBlob. pParse->aBlob might be dynamically +** allocated (if pParse->nBlobAlloc is greater than zero) in which case +** the caller is responsible for freeing the space allocated to pParse->aBlob +** when it has finished with it. Or pParse->aBlob might be a static string +** or a value obtained from sqlite3_value_blob(pArg). +** +** If the argument is a BLOB that is clearly not a JSONB, then this +** function might set an error message in ctx and return non-zero. +** It might also set an error message and return non-zero on an OOM error. +*/ +static int jsonFunctionArgToBlob( + sqlite3_context *ctx, + sqlite3_value *pArg, + JsonParse *pParse +){ + int eType = sqlite3_value_type(pArg); + static u8 aNull[] = { 0x00 }; + memset(pParse, 0, sizeof(pParse[0])); + switch( eType ){ + case SQLITE_NULL: { + pParse->aBlob = aNull; + pParse->nBlob = 1; + return 0; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(pArg) ){ + pParse->aBlob = (u8*)sqlite3_value_blob(pArg); + pParse->nBlob = sqlite3_value_bytes(pArg); + }else{ + sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); + return 1; + } + break; + } + case SQLITE_TEXT: { + const char *zJson = (const char*)sqlite3_value_text(pArg); + int nJson = sqlite3_value_bytes(pArg); + if( zJson==0 ) return 1; + if( sqlite3_value_subtype(pArg)==JSON_SUBTYPE ){ + pParse->zJson = (char*)zJson; + pParse->nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3_free(pParse->aBlob); + memset(pParse, 0, sizeof(pParse[0])); + return 1; + } + }else{ + jsonBlobAppendNodeType(pParse, JSONB_TEXTRAW, nJson); + jsonBlobAppendNBytes(pParse, (const u8*)zJson, nJson); + } + break; + } + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + int e = eType==SQLITE_INTEGER ? JSONB_INT : JSONB_FLOAT; + if( z==0 ) return 1; + jsonBlobAppendNodeType(pParse, e, n); + jsonBlobAppendNBytes(pParse, (const u8*)z, n); + break; + } + } + return 0; +} + +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments come in parse where each pair contains a JSON path and +** content to insert or set at that patch. Do the updates +** and return the result. +** +** The specific operation is determined by eEdit, which can be one +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +*/ +static void jsonInsertIntoBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv, + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ +){ + int i; + u32 rc; + const char *zPath = 0; + int flgs; + JsonParse px, ax; + + assert( (argc&1)==1 ); + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(argv[0]); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( px.aBlob==0 ) return; + for(i=1; i<argc-1; i+=2){ + const char *zPath = (const char*)sqlite3_value_text(argv[i]); + if( zPath==0 ) goto jsonInsertIntoBlob_patherror; + if( zPath[0]!='$' ) goto jsonInsertIntoBlob_patherror; + if( jsonFunctionArgToBlob(ctx, argv[i+1], &ax) ){ + break; + } + if( zPath[1]==0 ){ + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + return; /* return NULL if $ is removed */ + } + px.eEdit = eEdit; + px.nIns = ax.nBlob; + px.aIns = ax.aBlob; + rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); + if( ax.nBlobAlloc ) sqlite3_free(ax.aBlob); + if( rc==JSON_BLOB_NOTFOUND ) continue; + if( JSON_BLOB_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; + } + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + sqlite3_result_blob(ctx, px.aBlob, px.nBlob, + px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&px, 0, &s); + jsonReturnString(&s); + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + } + return; + +jsonInsertIntoBlob_patherror: + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonPathSyntaxError(zPath, ctx); + return; +} /**************************************************************************** ** SQL functions used for testing and debugging @@ -2395,7 +4528,7 @@ static void jsonParseFunc( printf("iSubst = %u\n", p->iSubst); printf("iHold = %u\n", p->iHold); jsonDebugPrintNodeEntries(p->aNode, p->nNode); - jsonReturnJson(p, p->aNode, ctx, 1, 0); + jsonReturnNodeAsJson(p, p->aNode, ctx, 1, 0); } /* @@ -2411,14 +4544,85 @@ static void jsonTest1Func( UNUSED_PARAMETER(argc); sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } + +/* SQL Function: jsonb_test2(BLOB_JSON) +** +** Render BLOB_JSON back into text. +** Development testing only. +*/ +static void jsonbTest2( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + const u8 *aBlob; + int nBlob; + UNUSED_PARAMETER(argc); + + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + nBlob = sqlite3_value_bytes(argv[0]); + jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); +} #endif /* SQLITE_DEBUG */ /**************************************************************************** ** Scalar SQL function implementations ****************************************************************************/ +/* SQL Function: jsonb(JSON) +** +** Convert the input argument into JSONB (the SQLite binary encoding of +** JSON). +** +** If the input is TEXT, or NUMERIC, try to parse it as JSON. If the fails, +** raise an error. Otherwise, return the resulting BLOB value. +** +** If the input is a BLOB, check to see if the input is a plausible +** JSONB. If it is, return it unchanged. Raise an error if it is not. +** Note that there could be internal inconsistencies in the BLOB - this +** routine does not do a full byte-for-byte validity check of the +** JSON blob. +** +** If the input is NULL, return NULL. +*/ +static void jsonbFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *pParse; + int nJson; + const char *zJson; + JsonParse x; + UNUSED_PARAMETER(argc); + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* No-op */ + }else if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_value(ctx, argv[0]); + }else{ + zJson = (const char*)sqlite3_value_text(argv[0]); + if( zJson==0 ) return; + nJson = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.zJson = (char*)zJson; + x.nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3_free(pParse->aBlob); + }else{ + sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + } + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + jsonParseReset(pParse); + } +} + /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. @@ -2431,9 +4635,9 @@ static void jsonQuoteFunc( JsonString jx; UNUSED_PARAMETER(argc); - jsonInit(&jx, ctx); - jsonAppendValue(&jx, argv[0]); - jsonResult(&jx); + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2450,14 +4654,14 @@ static void jsonArrayFunc( int i; JsonString jx; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=0; i<argc; i++){ jsonAppendSeparator(&jx); - jsonAppendValue(&jx, argv[i]); + jsonAppendSqlValue(&jx, argv[i]); } jsonAppendChar(&jx, ']'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2508,15 +4712,6 @@ static void jsonArrayLengthFunc( } /* -** Bit values for the flags passed into jsonExtractFunc() or -** jsonSetFunc() via the user-data value. -*/ -#define JSON_JSON 0x01 /* Result is always JSON */ -#define JSON_SQL 0x02 /* Result is always SQL */ -#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ -#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ - -/* ** json_extract(JSON, PATH, ...) ** "->"(JSON,PATH) ** "->>"(JSON,PATH) @@ -2548,6 +4743,10 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; + if( jsonFuncArgMightBeBinary(argv[0]) && argc==2 ){ + jsonExtractFromBlob(ctx, argv[0], argv[1], flags); + return; + } p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ @@ -2564,7 +4763,7 @@ static void jsonExtractFunc( ** LABEL ==> $.LABEL // PG compatible ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience */ - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); if( sqlite3Isdigit(zPath[0]) ){ jsonAppendRawNZ(&jx, "$[", 2); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); @@ -2574,27 +4773,27 @@ static void jsonExtractFunc( jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); jsonAppendChar(&jx, 0); } - pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); - jsonReset(&jx); + pNode = jx.eErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); + jsonStringReset(&jx); }else{ pNode = jsonLookup(p, zPath, 0, ctx); } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(p, pNode, ctx, 0, 0); + jsonReturnNodeAsJson(p, pNode, ctx, 0, 0); }else{ - jsonReturn(p, pNode, ctx, 1); + jsonReturnFromNode(p, pNode, ctx, 1); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx, 0); + if( p->nErr==0 && pNode ) jsonReturnFromNode(p, pNode, ctx, 0); } }else{ /* Two or more PATH arguments results in a JSON array with each ** element of the array being the value selected by one of the PATHs */ int i; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=1; i<argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2602,17 +4801,17 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(p, pNode, &jx); + jsonXlateNodeToText(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } } if( i==argc ){ jsonAppendChar(&jx, ']'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } - jsonReset(&jx); + jsonStringReset(&jx); } } @@ -2640,8 +4839,10 @@ static JsonNode *jsonMergePatch( for(i=1; i<pPatch->n; i += jsonNodeSize(&pPatch[i+1])+1){ u32 nKey; const char *zKey; - assert( pPatch[i].eType==JSON_STRING ); - assert( pPatch[i].jnFlags & JNODE_LABEL ); + if( pPatch[i].eType!=JSON_STRING ){ + pParse->nErr = 1; + return 0; + } assert( pPatch[i].eU==1 ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; @@ -2678,7 +4879,7 @@ static JsonNode *jsonMergePatch( pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; pParse->aNode[iRoot].u.iAppend = iStart; - VVA( pParse->aNode[iRoot].eU = 2 ); + JSON_VVA( pParse->aNode[iRoot].eU = 2 ); iRoot = iStart; pTarget = &pParse->aNode[iTarget]; } @@ -2710,13 +4911,15 @@ static void jsonPatchFunc( pX->useMod = 1; pY->useMod = 1; pResult = jsonMergePatch(pX, 0, pY->aNode); - assert( pResult!=0 || pX->oom ); - if( pResult && pX->oom==0 ){ + assert( pResult!=0 || pX->oom || pX->nErr ); + if( pX->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( pX->nErr ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else if( pResult ){ jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); - jsonReturnJson(pX, pResult, ctx, 0, 0); - }else{ - sqlite3_result_error_nomem(ctx); + jsonReturnNodeAsJson(pX, pResult, ctx, 0, 0); } } @@ -2741,12 +4944,12 @@ static void jsonObjectFunc( "of arguments", -1); return; } - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '{'); for(i=0; i<argc; i+=2){ if( sqlite3_value_type(argv[i])!=SQLITE_TEXT ){ sqlite3_result_error(ctx, "json_object() labels must be TEXT", -1); - jsonReset(&jx); + jsonStringReset(&jx); return; } jsonAppendSeparator(&jx); @@ -2754,10 +4957,10 @@ static void jsonObjectFunc( n = (u32)sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); - jsonAppendValue(&jx, argv[i+1]); + jsonAppendSqlValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2779,6 +4982,10 @@ static void jsonRemoveFunc( u32 i; if( argc<1 ) return; + if( jsonFuncArgMightBeBinary(argv[0]) ){ + jsonRemoveFromBlob(ctx, argc, argv); + return; + } pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ @@ -2793,7 +5000,7 @@ static void jsonRemoveFunc( } } if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); } remove_done: jsonDebugPrintParse(p); @@ -2864,7 +5071,10 @@ static void jsonReplaceNode( k = jsonParseAddNode(p, JSON_STRING, n, zCopy); assert( k>0 || p->oom ); if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; - }else{ + break; + } + replace_with_json: + { JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); if( pPatch==0 ){ p->oom = 1; @@ -2881,9 +5091,13 @@ static void jsonReplaceNode( break; } default: { - jsonParseAddNode(p, JSON_NULL, 0, 0); - sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); - p->nErr++; + if( jsonFuncArgMightBeBinary(pValue) ){ + goto replace_with_json; + }else{ + jsonParseAddNode(p, JSON_NULL, 0, 0); + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + } break; } } @@ -2910,6 +5124,10 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } + if( jsonFuncArgMightBeBinary(argv[0]) && argc>=3 ){ + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); + return; + } pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); if( pParse==0 ) return; pParse->nJPRef++; @@ -2922,7 +5140,7 @@ static void jsonReplaceFunc( jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); replace_err: jsonDebugPrintParse(pParse); jsonParseFree(pParse); @@ -2951,7 +5169,8 @@ static void jsonSetFunc( const char *zPath; u32 i; int bApnd; - int bIsSet = sqlite3_user_data(ctx)!=0; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; if( (argc&1)==0 ) { @@ -2976,7 +5195,7 @@ static void jsonSetFunc( } } jsonDebugPrintParse(pParse); - jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); jsonSetDone: jsonParseFree(pParse); } @@ -3013,8 +5232,14 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a well-formed canonical JSON string according -** to RFC-7159. Return 0 otherwise. +** Return 1 if the argument is one of: +** +** * A well-formed canonical JSON string according to RFC-8259 +** (without JSON5 enhancements), or +** +** * A BLOB that plausibly could be a JSONB value. +** +** Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -3030,6 +5255,10 @@ static void jsonValidFunc( #endif return; } + if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_int(ctx, 1); + return; + } p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); @@ -3113,24 +5342,34 @@ static void jsonArrayStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '['); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - jsonAppendValue(pStr, argv[0]); + jsonAppendSqlValue(pStr, argv[0]); } } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -3219,7 +5458,7 @@ static void jsonObjectStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); @@ -3229,17 +5468,28 @@ static void jsonObjectStep( n = (u32)sqlite3_value_bytes(argv[0]); jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); + jsonAppendSqlValue(pStr, argv[1]); } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -3393,7 +5643,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ if( pUp->eType==JSON_ARRAY ){ assert( pUp->eU==0 || pUp->eU==3 ); testcase( pUp->eU==3 ); - VVA( pUp->eU = 3 ); + JSON_VVA( pUp->eU = 3 ); if( iUp==p->i-1 ){ pUp->u.iKey = 0; }else{ @@ -3429,26 +5679,28 @@ static void jsonAppendObjectPathElement( JsonString *pStr, JsonNode *pNode ){ - int jj, nn; + int nn; const char *z; + int bNeedQuote = 0; assert( pNode->eType==JSON_STRING ); assert( pNode->jnFlags & JNODE_LABEL ); assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - if( (pNode->jnFlags & JNODE_RAW)==0 ){ - assert( nn>=2 ); - assert( z[0]=='"' || z[0]=='\'' ); - assert( z[nn-1]=='"' || z[0]=='\'' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){} - if( jj==nn-1 ){ - z++; - nn -= 2; - } - } + if( pNode->jnFlags & JNODE_RAW ){ + /* no-op */ + }else if( nn==0 || !sqlite3Isalpha(z[0]) ){ + bNeedQuote = 1; + }else{ + int jj; + for(jj=1; jj<nn && sqlite3Isalnum(z[jj]); jj++){} + bNeedQuote = jj<nn; + } + if( bNeedQuote ){ + jsonPrintf(nn+4, pStr, ".\"%.*s\"", nn, z); + }else{ + jsonPrintf(nn+2, pStr, ".%.*s", nn, z); } - jsonPrintf(nn+2, pStr, ".%.*s", nn, z); } /* Append the name of the path for element i to pStr @@ -3491,7 +5743,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse, pThis, ctx, 0); + jsonReturnFromNode(&p->sParse, pThis, ctx, 0); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -3507,7 +5759,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(&p->sParse, pThis, ctx, 0); + jsonReturnFromNode(&p->sParse, pThis, ctx, 0); break; } case JEACH_TYPE: { @@ -3518,7 +5770,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(&p->sParse, pThis, ctx, 0); + jsonReturnFromNode(&p->sParse, pThis, ctx, 0); break; } case JEACH_ID: { @@ -3534,7 +5786,7 @@ static int jsonEachColumn( } case JEACH_FULLKEY: { JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); if( p->bRecursive ){ jsonEachComputePath(p, &x, p->i); }else{ @@ -3549,15 +5801,15 @@ static int jsonEachColumn( jsonAppendObjectPathElement(&x, pThis); } } - jsonResult(&x); + jsonReturnString(&x); break; } case JEACH_PATH: { if( p->bRecursive ){ JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); + jsonReturnString(&x); break; } /* For json_each() path and root are the same so fall through @@ -3571,8 +5823,12 @@ static int jsonEachColumn( break; } case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } break; } } @@ -3666,12 +5922,19 @@ static int jsonEachFilter( const char *z; const char *zRoot = 0; sqlite3_int64 n; + int isBinary; UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + z = (const char*)sqlite3_value_blob(argv[0]); + isBinary = 1; + }else{ + z = (const char*)sqlite3_value_text(argv[0]); + isBinary = 0; + } if( z==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; @@ -3681,9 +5944,11 @@ static int jsonEachFilter( n = sqlite3_value_bytes(argv[0]); p->sParse.zJson = sqlite3RCStrNew( n+1 ); if( p->sParse.zJson==0 ) return SQLITE_NOMEM; - memcpy(p->sParse.zJson, z, (size_t)n+1); + memcpy(p->sParse.zJson, z, (size_t)n+(isBinary==0)); + p->sParse.nJson = n; } p->sParse.bJsonIsRCStr = 1; + p->sParse.isBinary = isBinary; p->zJson = p->sParse.zJson; if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; @@ -3714,7 +5979,7 @@ static int jsonEachFilter( } if( zErr ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; }else if( pNode==0 ){ @@ -3727,7 +5992,7 @@ static int jsonEachFilter( p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ assert( pNode->eU==0 ); - VVA( pNode->eU = 3 ); + JSON_VVA( pNode->eU = 3 ); pNode->u.iKey = 0; p->iEnd = p->i + pNode->n + 1; if( p->bRecursive ){ @@ -3811,41 +6076,56 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - /* calls sqlite3_result_subtype() */ - /* | */ - /* Uses cache ______ | __ calls sqlite3_value_subtype() */ - /* | | | */ - /* Num args _________ | | | ___ Flags */ - /* | | | | | */ - /* | | | | | */ - JFUNCTION(json, 1, 1, 1, 0, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, 1, 1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 1, 0, 0, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 1, 0, 0, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 1, 0, 0, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 1, 1, 0, 0, jsonExtractFunc), - JFUNCTION(->, 2, 1, 1, 0, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, 1, 0, 0, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 1, 1, 1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, 1, 1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 1, 1, 0, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, 1, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 1, 1, 0, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 1, 1, 1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, 1, 1, 1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 1, 0, 0, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 1, 0, 0, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 1, 0, 0, 0, jsonValidFunc), + /* sqlite3_result_subtype() ----, ,--- sqlite3_value_subtype() */ + /* | | */ + /* Uses cache ------, | | ,---- Returns JSONB */ + /* | | | | */ + /* Number of arguments ---, | | | | ,--- Flags */ + /* | | | | | | */ + JFUNCTION(json, 1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonbFunc), + JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), + JFUNCTION(json_extract, -1,1,1, 0,0,0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1,1,0, 0,1,0, jsonExtractFunc), + JFUNCTION(->, 2,1,1, 0,0,JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2,1,0, 0,0,JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1,1,1, 1,0,0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1,1,0, 1,1,0, jsonSetFunc), + JFUNCTION(json_object, -1,0,1, 1,0,0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), + JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), + JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_replace, -1,1,1, 1,0,0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1,1,0, 1,1,0, jsonReplaceFunc), + JFUNCTION(json_set, -1,1,1, 1,0,JSON_ISSET, jsonSetFunc), + JFUNCTION(jsonb_set, -1,1,0, 1,1,JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_type, 2,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_valid, 1,1,0, 0,0,0, jsonValidFunc), #if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 1, 1, 0, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 1, 0, 1, 0, jsonTest1Func), + JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), + JFUNCTION(json_test1, 1,1,0, 1,0,0, jsonTest1Func), + JFUNCTION(jsonb_test2, 1,1,0, 0,1,0, jsonbTest2), #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| SQLITE_DETERMINISTIC) }; |