diff options
Diffstat (limited to 'src/json.c')
| -rw-r--r-- | src/json.c | 2284 |
1 files changed, 2059 insertions, 225 deletions
diff --git a/src/json.c b/src/json.c index 253fce9f4..955c35377 100644 --- a/src/json.c +++ b/src/json.c @@ -15,19 +15,128 @@ ** This file began as an extension in ext/misc/json1.c in 2015. That ** extension proved so useful that it has now been moved into the core. ** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.44.0 (pending), these routines also accept +** BLOB values that have JSON encoded using a binary representation we +** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk +** format SQLite JSONB is completely different and incompatible with +** PostgreSQL JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, @@ -74,11 +183,12 @@ static const char jsonIsOk[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - +/* Put code used only for testing inside the JSON_VVA() macro. +*/ #if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define VVA(X) +# define JSON_VVA(X) #else -# define VVA(X) X +# define JSON_VVA(X) X #endif /* Objects */ @@ -110,7 +220,7 @@ struct JsonCleanup { void *pArg; /* Argument to xOp() */ }; -/* JSON type values +/* JSON type values for JsonNode.eType */ #define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ #define JSON_NULL 1 @@ -122,17 +232,18 @@ struct JsonCleanup { #define JSON_ARRAY 7 #define JSON_OBJECT 8 -/* The "subtype" set for JSON values */ -#define JSON_SUBTYPE 74 /* Ascii for "J" */ - -/* -** Names of the various JSON types: +/* Human-readalbe names for the JsonNode types: */ static const char * const jsonType[] = { "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* Bit values for the JsonNode.jnFlag field */ #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ @@ -143,6 +254,16 @@ static const char * const jsonType[] = { #define JNODE_LABEL 0x20 /* Is a label of an object */ #define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ +/* +** Bit values for the flags passed into jsonExtractFunc() or +** jsonSetFunc() via the user-data value. +*/ +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ + /* A single node of parsed JSON. An array of these nodes describes ** a parse of JSON + edits. @@ -204,12 +325,17 @@ struct JsonParse { u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ + u8 isBinary; /* True if zJson is the binary encoding */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ u32 iHold; /* Age of this entry in the cache for LRU replacement */ + /* Binary format */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ + u8 *aBlob; /* BLOB representation of zJson */ }; /* @@ -222,12 +348,24 @@ struct JsonParse { #define JSON_MAX_DEPTH 1000 /************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static void jsonRenderNodeAsBlob(JsonParse*,JsonNode*,JsonParse*); +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); +static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); +static u32 jsonRenderBlob(JsonParse*,u32,JsonString*); + +/************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ -/* Set the JsonString object to an empty string +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. */ -static void jsonZero(JsonString *p){ +static void jsonStringZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; @@ -236,39 +374,39 @@ static void jsonZero(JsonString *p){ /* Initialize the JsonString object */ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; p->bErr = 0; - jsonZero(p); + jsonStringZero(p); } /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(JsonString *p){ +static void jsonStringReset(JsonString *p){ if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); - jsonZero(p); + jsonStringZero(p); } /* Report an out-of-memory (OOM) condition */ -static void jsonOom(JsonString *p){ +static void jsonStringOom(JsonString *p){ p->bErr = 1; sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); + jsonStringReset(p); } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(JsonString *p, u32 N){ +static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ if( p->bErr ) return 1; zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ - jsonOom(p); + jsonStringOom(p); return SQLITE_NOMEM; } memcpy(zNew, p->zBuf, (size_t)p->nUsed); @@ -278,7 +416,7 @@ static int jsonGrow(JsonString *p, u32 N){ p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); if( p->zBuf==0 ){ p->bErr = 1; - jsonZero(p); + jsonStringZero(p); return SQLITE_NOMEM; } } @@ -288,29 +426,29 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static SQLITE_NOINLINE void jsonAppendExpand( +static SQLITE_NOINLINE void jsonStringExpandAndAppend( JsonString *p, const char *zIn, u32 N ){ assert( N>0 ); - if( jsonGrow(p,N) ) return; + if( jsonStringGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } } static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ - assert( N>0 ); + if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -322,7 +460,7 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; va_start(ap, zFormat); sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); va_end(ap); @@ -332,7 +470,7 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ - if( jsonGrow(p,1) ) return; + if( jsonStringGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } static void jsonAppendChar(JsonString *p, char c){ @@ -355,7 +493,7 @@ static int jsonForceRCStr(JsonString *p){ if( p->bStatic==0 ) return 1; p->nAlloc = 0; p->nUsed++; - jsonGrow(p, p->nUsed); + jsonStringGrow(p, p->nUsed); p->nUsed--; return p->bStatic==0; } @@ -379,7 +517,8 @@ static void jsonAppendSeparator(JsonString *p){ */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 i; - if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; + if( zIn==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; for(i=0; i<N; i++){ unsigned char c = ((unsigned const char*)zIn)[i]; @@ -387,7 +526,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = c; }else if( c=='"' || c=='\\' ){ json_simple_escape: - if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + if( (p->nUsed+N+3-i > p->nAlloc) && jsonStringGrow(p,N+3-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ @@ -408,7 +547,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ c = aSpecial[c]; goto json_simple_escape; } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + if( (p->nUsed+N+7+i > p->nAlloc) && jsonStringGrow(p,N+7-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = 'u'; p->zBuf[p->nUsed++] = '0'; @@ -429,8 +568,6 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ u32 i; jsonAppendChar(p, '"'); - zIn++; - N -= 2; while( N>0 ){ for(i=0; i<N && zIn[i]!='\\'; i++){} if( i>0 ){ @@ -487,6 +624,14 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + char *zBuf = sqlite3_malloc64( N+1 ); + if( zBuf==0 ){ + p->bErr = 1; + return; + } + memcpy(zBuf, zIn, N); + zBuf[N] = 0; + zIn = zBuf; if( zIn[0]=='+' ){ zIn++; N--; @@ -504,10 +649,11 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ assert( rc==2 ); jsonAppendRawNZ(p, "9.0e999", 7); } - return; + }else{ + assert( N>0 ); + jsonAppendRawNZ(p, zIn, N); } - assert( N>0 ); - jsonAppendRawNZ(p, zIn, N); + sqlite3_free(zBuf); } /* @@ -546,10 +692,10 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ /* -** Append a function parameter value to the JSON string under -** construction. +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. */ -static void jsonAppendValue( +static void jsonAppendSqlValue( JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ @@ -579,10 +725,16 @@ static void jsonAppendValue( break; } default: { - if( p->bErr==0 ){ + if( jsonFuncArgMightBeBinary(pValue) ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(pValue); + px.nBlob = sqlite3_value_bytes(pValue); + jsonRenderBlob(&px, 0, p); + }else if( p->bErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->bErr = 2; - jsonReset(p); + jsonStringReset(p); } break; } @@ -590,13 +742,17 @@ static void jsonAppendValue( } -/* Make the JSON in p the result of the SQL function. +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. ** -** The JSON string is reset. +** The JsonString is reset. */ -static void jsonResult(JsonString *p){ +static void jsonReturnString(JsonString *p){ if( p->bErr==0 ){ - if( p->bStatic ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ @@ -609,7 +765,7 @@ static void jsonResult(JsonString *p){ if( p->bErr==1 ){ sqlite3_result_error_nomem(p->pCtx); } - jsonReset(p); + jsonStringReset(p); } /************************************************************************** @@ -708,7 +864,7 @@ static int jsonParseAddCleanup( ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static void jsonRenderNode( +static void jsonRenderNodeAsText( JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ JsonString *pOut /* Write JSON here */ @@ -756,8 +912,9 @@ static void jsonRenderNode( }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ - assert( pNode->n>0 ); + jsonAppendChar(pOut, '"'); jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); } break; } @@ -788,7 +945,7 @@ static void jsonRenderNode( while( j<=pNode->n ){ if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonRenderNodeAsText(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } @@ -808,9 +965,9 @@ static void jsonRenderNode( while( j<=pNode->n ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonRenderNodeAsText(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(pParse, &pNode[j+1], pOut); + jsonRenderNodeAsText(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -827,27 +984,47 @@ static void jsonRenderNode( } /* -** Return a JsonNode and all its descendants as a JSON string. +** Make the return value of an SQL function be the JSON encoded by pNode. +** +** By default, the node is rendered as RFC-8259 JSON text (canonical +** JSON text without any JSON-5 enhancements). However if the +** JSON_BLOB flag is set in the user-data for the function, then the +** node is rendered into the JSONB format and returned as a BLOB. */ -static void jsonReturnJson( +static void jsonReturnNodeAsJson( JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ int bGenerateAlt /* Also store the rendered text in zAlt */ ){ + int flags; JsonString s; if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); return; } - if( pParse->nErr==0 ){ - jsonInit(&s, pCtx); - jsonRenderNode(pParse, pNode, &s); + if( pParse->nErr ){ + return; + } + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + JsonParse x; + memset(&x, 0, sizeof(x)); + jsonRenderNodeAsBlob(pParse, pNode, &x); + if( x.oom ){ + sqlite3_result_error_nomem(pCtx); + sqlite3_free(x.aBlob); + }else{ + sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); + } + }else{ + jsonStringInit(&s, pCtx); + jsonRenderNodeAsText(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ pParse->zAlt = sqlite3RCStrRef(s.zBuf); pParse->nAlt = s.nUsed; } - jsonResult(&s); + jsonReturnString(&s); sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } } @@ -884,9 +1061,16 @@ static u32 jsonHexToInt4(const char *z){ } /* -** Make the JsonNode the return value of the function. +** Make the return value from an SQL function be the SQL value of +** JsonNode pNode. +** +** If pNode is an atom (not an array or object) then the value returned +** is a pure SQL value - an SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT, or +** SQLITE_NULL. However, if pNode is a JSON array or object, then the +** value returned is either RFC-8259 JSON text or a BLOB in the JSONB +** format, depending on the JSON_BLOB flag of the function user-data. */ -static void jsonReturn( +static void jsonReturnNodeAsSql( JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx /* Return value for this function */ @@ -910,12 +1094,20 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; + char *zz; + sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( pNode->eU==1 ); - z = pNode->u.zJContent; + zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); + if( zz==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + z = zz; if( z[0]=='-' ){ z++; bNeg = 1; } else if( z[0]=='+' ){ z++; } rc = sqlite3DecOrHexToI64(z, &i); + sqlite3DbFree(db, zz); if( rc<=1 ){ sqlite3_result_int64(pCtx, bNeg ? -i : i); }else if( rc==3 && bNeg ){ @@ -943,7 +1135,7 @@ static void jsonReturn( }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ @@ -960,7 +1152,7 @@ static void jsonReturn( sqlite3_result_error_nomem(pCtx); break; } - for(i=1, j=0; i<n-1; i++){ + for(i=0, j=0; i<n; i++){ char c = z[i]; if( c=='\\' ){ c = z[++i]; @@ -1035,15 +1227,12 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx, 0); + jsonReturnNodeAsJson(pParse, pNode, pCtx, 0); break; } } } -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - /* ** A macro to hint to the compiler that a function should not be ** inlined. @@ -1106,7 +1295,7 @@ static int jsonParseAddNode( assert( p!=0 ); p->eType = (u8)(eType & 0xff); p->jnFlags = (u8)(eType >> 8); - VVA( p->eU = zContent ? 1 : 0 ); + JSON_VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; return pParse->nNode++; @@ -1331,7 +1520,7 @@ static const struct NanInfName { ** -4 ',' seen ** -5 ':' seen */ -static int jsonParseValue(JsonParse *pParse, u32 i){ +static int jsonParseValueFromText(JsonParse *pParse, u32 i){ char c; u32 j; int iThis; @@ -1350,7 +1539,7 @@ json_parse_restart: } for(j=i+1;;j++){ u32 nNode = pParse->nNode; - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; @@ -1393,7 +1582,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1401,7 +1590,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1420,7 +1609,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1433,7 +1622,9 @@ json_parse_restart: pParse->iErr = j; return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } pParse->iDepth--; return j+1; } @@ -1447,7 +1638,7 @@ json_parse_restart: } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1471,7 +1662,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1530,7 +1721,7 @@ json_parse_restart: return -1; } } - jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); + jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j-1-i, &z[i+1]); return j+1; } case 't': { @@ -1648,7 +1839,10 @@ json_parse_restart: } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') + && ALWAYS(j-2>=i) + && sqlite3Isdigit(z[j-2]) + ){ pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; }else{ @@ -1762,13 +1956,42 @@ json_parse_restart: } /* End switch(z[i]) */ } +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + /* -** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory held by pParse, -** but not pParse itself. +** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB +** blob) into the JsonNode representation. ** -** pParse must be initialized to an empty parse object prior to calling -** this routine. +** Return 0 on success or non-zero if there are any errors. +** If an error occurs, free all memory held by pParse, but not pParse itself. +** +** pParse must be initialized with pParse->zJson set to the input text or +** blob prior to calling this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ @@ -1776,9 +1999,15 @@ static int jsonParse( ){ int i; const char *zJson = pParse->zJson; - i = jsonParseValue(pParse, 0); + if( pParse->isBinary ){ + pParse->aBlob = (u8*)pParse->zJson; + pParse->nBlob = pParse->nJson; + i = jsonParseValueFromBlob(pParse, 0); + }else{ + i = jsonParseValueFromText(pParse, 0); + } if( pParse->oom ) i = -1; - if( i>0 ){ + if( !pParse->isBinary && i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; if( zJson[i] ){ @@ -1804,34 +2033,6 @@ static int jsonParse( return 0; } - -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - /* ** Compute the parentage of all nodes in a completed parse. */ @@ -1887,8 +2088,8 @@ static JsonParse *jsonParseCached( sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ int bUnedited /* No prior edits allowed */ ){ - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); + char *zJson; + int nJson; JsonParse *p; JsonParse *pMatch = 0; int iKey; @@ -1896,6 +2097,16 @@ static JsonParse *jsonParseCached( u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; int bJsonRCStr; + int isBinary; + + if( jsonFuncArgMightBeBinary(pJson) ){ + zJson = (char*)sqlite3_value_blob(pJson); + isBinary = 1; + }else{ + zJson = (char*)sqlite3_value_text(pJson); + isBinary = 0; + } + nJson = sqlite3_value_bytes(pJson); if( zJson==0 ) return 0; for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){ @@ -1954,9 +2165,11 @@ static JsonParse *jsonParseCached( p->bJsonIsRCStr = 1; }else{ p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+1); + memcpy(p->zJson, zJson, nJson+(isBinary==0)); } p->nJPRef = 1; + p->isBinary = isBinary; + p->nJson = nJson; if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; @@ -1966,7 +2179,6 @@ static JsonParse *jsonParseCached( jsonParseFree(p); return 0; } - p->nJson = nJson; p->iHold = iMaxHold+1; /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, @@ -1980,13 +2192,8 @@ static JsonParse *jsonParseCached( */ static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; } static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ if( p1->jnFlags & JNODE_RAW ){ @@ -1998,9 +2205,6 @@ static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ } } -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such @@ -2097,7 +2301,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; } return pNode; @@ -2116,7 +2320,9 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + i++; + } j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; @@ -2178,7 +2384,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); } return pNode; } @@ -2298,6 +2504,1509 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ } } +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + + +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u32 t; + if( N<=pParse->nBlobAlloc ) return 0; + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; + } + if( t<N ) t = N+100; + aNew = sqlite3_realloc64( pParse->aBlob, t ); + if( aNew==0 ){ pParse->oom = 1; return 1; } + pParse->aBlob = aNew; + pParse->nBlobAlloc = t; + return 0; +} + +/* Expand pParse->aBlob and append N bytes. +** +** Return the number of errors. +*/ +static SQLITE_NOINLINE int jsonBlobExpandAndAppend( + JsonParse *pParse, + const u8 *aData, + u32 N +){ + if( jsonBlobExpand(pParse, pParse->nBlob+N) ) return 1; + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append a single character. Return 1 if an error occurs. +*/ +static int jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, &c, 1); + } + pParse->aBlob[pParse->nBlob++] = c; + return 0; +} + +/* Append bytes. Return 1 if an error occurs. +*/ +static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ + if( pParse->nBlob+N > pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, aData, N); + } + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append an node type byte together with the payload size. +*/ +static void jsonBlobAppendNodeType( + JsonParse *pParse, + u8 eType, + u32 szPayload +){ + u8 a[5]; + if( szPayload<=11 ){ + jsonBlobAppendOneByte(pParse, eType | (szPayload<<4)); + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 2); + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 3); + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 5); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static void jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a; + u8 szType; + if( pParse->oom ) return; + a = &pParse->aBlob[i]; + szType = a[0]>>4; + if( szType<=11 ){ + assert( szPayload<=11 ); + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( szType==0xc ){ + assert( szPayload<=0xff ); + assert( i+1<pParse->nBlob ); + a[1] = szPayload & 0xff; + }else if( szType==0xd ){ + assert( szPayload<=0xffff ); + assert( i+2<pParse->nBlob ); + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + assert( szType==0xe ); + assert( i+4<pParse->nBlob ); + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } +} + +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !sqlite3Isxdigit(z[1]) ) return 0; + if( !sqlite3Isxdigit(z[2]) ) return 0; + if( !sqlite3Isxdigit(z[3]) ) return 0; + if( !sqlite3Isxdigit(z[4]) ) return 0; + *pOp = JSONB_TEXTJ; + return 1; +} + +/* +** Parse a single JSON text value which begins at pParse->zJson[i] into +** its equivalent BLOB representation in pParse->aBlob[]. The parse is +** appended to pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. +** +** Return the index of the first character past the end of the value parsed, +** or one of the following special result codes: +** +** 0 End of input +** -1 Syntax error +** -2 '}' seen +** -3 ']' seen +** -4 ',' seen +** -5 ':' seen +*/ +static int jsonTranslateTextValueToBlob(JsonParse *pParse, u32 i){ + char c; + u32 j; + u32 iThis, iStart; + int x; + u8 t; + const char *z = pParse->zJson; +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { + /* Parse object */ + iThis = pParse->nBlob; + jsonBlobAppendNodeType(pParse, JSONB_OBJECT, (pParse->nJson-i)*2); + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + iStart = pParse->nBlob; + for(j=i+1;;j++){ + u32 iBlob = pParse->nBlob; + x = jsonTranslateTextValueToBlob(pParse, j); + if( x<=0 ){ + int op; + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) + ){ + k++; + } + assert( iBlob==pParse->nBlob ); + jsonBlobAppendNodeType(pParse, op, k-j); + jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } + } + if( pParse->oom ) return -1; + t = pParse->aBlob[iBlob] & 0x0f; + if( t<JSONB_TEXT || t>JSONB_TEXTRAW ){ + pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==':' ){ + j++; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonTranslateTextValueToBlob(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: + x = jsonTranslateTextValueToBlob(pParse, j); + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonTranslateTextValueToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + if( pParse->oom==0 ){ + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + } + pParse->iDepth--; + return j+1; + } + case '[': { + /* Parse array */ + iThis = pParse->nBlob; + jsonBlobAppendNodeType(pParse, JSONB_ARRAY, pParse->nJson - i); + iStart = pParse->nBlob; + if( pParse->oom ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + for(j=i+1;;j++){ + x = jsonTranslateTextValueToBlob(pParse, j); + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonTranslateTextValueToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + if( pParse->oom==0 ){ + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + } + pParse->iDepth--; + return j+1; + } + case '\'': { + u8 opcode; + char cDelim; + pParse->hasNonstd = 1; + opcode = JSONB_TEXT; + goto parse_string; + case '"': + /* Parse string */ + opcode = JSONB_TEXT; + parse_string: + cDelim = z[i]; + for(j=i+1; 1; j++){ + if( jsonIsOk[(unsigned char)z[j]] ) continue; + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else{ + pParse->iErr = j; + return -1; + } + }else if( c<=0x1f ){ + /* Control characters are not allowed in strings */ + pParse->iErr = j; + return -1; + } + } + jsonBlobAppendNodeType(pParse, opcode, j-1-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i+1], j-1-i); + return j+1; + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_TRUE); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonBlobAppendOneByte(pParse, JSONB_FALSE); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenE; + pParse->hasNonstd = 1; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + seenE = 0; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + /* Parse number */ + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + parse_number: + seenE = 0; + assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + + if( c<='0' ){ + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( t==0x00 ); + pParse->hasNonstd = 1; + t = 0x01; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 6); + jsonBlobAppendNBytes(pParse, (const u8*)"-9e999", 6); + }else{ + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 5); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + t |= 0x03; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } + } + parse_number_2: + for(j=i+1;; j++){ + c = z[j]; + if( sqlite3Isdigit(c) ) continue; + if( c=='.' ){ + if( (t & 0x02)!=0 ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } + continue; + } + break; + } + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; + jsonBlobAppendNodeType(pParse, JSONB_INT+t, j-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); + return j; + } + case '}': { + pParse->iErr = i; + return -2; /* End of {...} */ + } + case ']': { + pParse->iErr = i; + return -3; /* End of [...] */ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { + return 0; /* End of file */ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; k<sizeof(aNanInfName)/sizeof(aNanInfName[0]); k++){ + if( c!=aNanInfName[k].c1 && c!=aNanInfName[k].c2 ) continue; + nn = aNanInfName[k].n; + if( sqlite3StrNICmp(&z[i], aNanInfName[k].zMatch, nn)!=0 ){ + continue; + } + if( sqlite3Isalnum(z[i+nn]) ) continue; + if( aNanInfName[k].eType==JSON_REAL ){ + jsonBlobAppendOneByte(pParse, JSONB_FLOAT); + jsonBlobAppendOneByte(pParse, 5); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + }else{ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + } + pParse->hasNonstd = 1; + return i + nn; + } + pParse->iErr = i; + return -1; /* Syntax error */ + } + } /* End switch(z[i]) */ +} + + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. +** +** pParse must be initialized to an empty parse object prior to calling +** this routine. +*/ +static int jsonConvertTextToBlob( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx /* Report errors here */ +){ + int i; + const char *zJson = pParse->zJson; + i = jsonTranslateTextValueToBlob(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( fast_isspace(zJson[i]) ) i++; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. +*/ +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + (void)jsonTranslateTextValueToBlob(&px, 0); + if( px.oom ){ + sqlite3_free(px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, sqlite3_free); + } +} + +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. Return 0 on error. +*/ +static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + if( i>pParse->nBlob ){ + *pSz = 0; + return 0; + } + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else if( x==13 ){ + if( i+2>pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else{ + if( i+4>pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + } + if( i+sz+n>pParse->nBlob ){ + sz = 0; + n = 0; + } + *pSz = sz; + return n; +} + + +/* +** Convert the binary BLOB representation of JSON beginning at +** aBlob[0] (and extending for no more than nBlob bytes) into +** a pure JSON string. The string is appended to pOut. +*/ +static u32 jsonRenderBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + u32 i, /* Start rendering at this index */ + JsonString *pOut /* Write JSON here */ +){ + u32 sz, n, j, iEnd; + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->bErr = 1; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + jsonAppendRawNZ(pOut, "null", 4); + return i+1; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + return i+1; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + return i+1; + } + case JSONB_INT: + case JSONB_FLOAT: { + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + int k; + sqlite3_uint64 u = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( zIn[0]=='-' ){ + zIn++; + sz--; + jsonAppendChar(pOut, '-'); + } + for(k=2; k<sz; k++){ + u = u*16 + sqlite3HexToInt(zIn[k]); + } + jsonPrintf(100,pOut,"%llu",u); + break; + } + case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ + int k; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( zIn[0]=='-' ){ + zIn++; + sz--; + jsonAppendChar(pOut, '-'); + } + if( zIn[0]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(k=0; k<sz; k++){ + if( zIn[k]=='.' && (k+1==sz || !sqlite3Isdigit(zIn[k+1])) ){ + k++; + jsonAppendRaw(pOut, zIn, k); + zIn += k; + sz -= k; + jsonAppendChar(pOut, '0'); + break; + } + } + if( sz>0 ){ + jsonAppendRawNZ(pOut, zIn, sz); + } + break; + } + case JSONB_TEXT: + case JSONB_TEXTJ: { + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz>0 ){ + for(k=0; k<sz && zIn[k]!='\\'; k++){} + if( k>0 ){ + jsonAppendRawNZ(pOut, zIn, k); + zIn += k; + sz -= k; + if( sz==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u0009", 6); + break; + case 'x': + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( zIn[2]=='\n' ){ + zIn++; + sz--; + } + break; + case '\n': + break; + case 0xe2: + assert( sz>=4 ); + assert( 0x80==(u8)zIn[2] ); + assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + zIn += 2; + sz -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + zIn += 2; + sz -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + j = i+n; + iEnd = j+sz; + while( j<iEnd ){ + j = jsonRenderBlob(pParse, j, pOut); + jsonAppendChar(pOut, ','); + } + if( sz>0 ) pOut->nUsed--; + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + j = i+n; + iEnd = j+sz; + while( j<iEnd ){ + j = jsonRenderBlob(pParse, j, pOut); + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + if( sz>0 ) pOut->nUsed--; + jsonAppendChar(pOut, '}'); + break; + } + + default: { + break; + } + } + return i+n+sz; +} + +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + aBlob = sqlite3_value_blob(pJson); + if( (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + return sz+n==(u32)nBlob; +} + +/* Parse a single element of binary JSON into the legacy Node structure. +** Return the index of the first byte past the end of the binary JSON element. +** +** This routine is a temporary translator while the legacy Node encoding +** is still in use. It will be removed after all processing translates +** to the new BLOB encoding. +*/ +static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ + u8 t; /* Node type */ + u32 sz; /* Node size */ + u32 x; /* Index of payload start */ + + const char *zPayload; + x = jsonbPayloadSize(pParse, i, &sz); + if( x==0 ) return -1; + t = pParse->zJson[i] & 0x0f; + zPayload = &pParse->zJson[i+x]; + switch( t ){ + case JSONB_NULL: { + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + break; + } + case JSONB_TRUE: { + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + break; + } + case JSONB_FALSE: { + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + break; + } + case JSONB_INT: { + jsonParseAddNode(pParse, JSON_INT, sz, zPayload); + break; + } + case JSONB_INT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_FLOAT: { + jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); + break; + } + case JSONB_FLOAT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_TEXT: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXTJ: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_ESCAPE<<8), sz, zPayload); + break; + } + case JSONB_TEXT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_STRING | ((JNODE_ESCAPE|JNODE_JSON5)<<8), + sz, zPayload); + break; + } + case JSONB_TEXTRAW: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_ARRAY: { + int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + u32 j = i+x; + while( j<i+x+sz ){ + int r = jsonParseValueFromBlob(pParse, j); + if( r<=0 ) return -1; + j = (u32)r; + } + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } + break; + } + case JSONB_OBJECT: { + int iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + u32 j = i+x, k = 0; + while( j<i+x+sz ){ + int r = jsonParseValueFromBlob(pParse, j); + if( r<=0 ) return -1; + if( (k++&1)==0 && !pParse->oom ){ + pParse->aNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; + } + j = (u32)r; + } + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } + break; + } + } + return i+x+sz; +} + +/* +** Convert pNode that belongs to pParse into the BLOB representation. +** The BLOB representation is added to the pOut->aBlob[] array. +*/ +static void jsonRenderNodeAsBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + JsonNode *pNode, /* The node to render */ + JsonParse *pOut /* Write the BLOB rendering of JSON here */ +){ + assert( pNode!=0 ); + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( i<pParse->nNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrev<i ); + if( pParse->aNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonBlobAppendNodeType(pOut, JSONB_NULL, 0); + break; + } + case JSON_TRUE: { + jsonBlobAppendNodeType(pOut, JSONB_TRUE, 0); + break; + } + case JSON_FALSE: { + jsonBlobAppendNodeType(pOut, JSONB_FALSE, 0); + break; + } + case JSON_STRING: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_RAW ){ + if( memchr(pNode->u.zJContent, '"', pNode->n)==0 + && memchr(pNode->u.zJContent, '\\', pNode->n)==0 + ){ + op = JSONB_TEXT; + }else{ + op = JSONB_TEXTRAW; + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_TEXT5; + }else{ + op = JSONB_TEXTJ; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_REAL: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_FLOAT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_FLOAT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_INT: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_INT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_INT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_ARRAY, pParse->nJson*2); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + case JSON_OBJECT: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_OBJECT, pParse->nJson*2); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); + jsonRenderNodeAsBlob(pParse, &pNode[j+1], pOut); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + } +} + +/* +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. +*/ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; n>0 && i<iEnd; i+=sz+n, k++){ + n = jsonbPayloadSize(pParse, i, &sz); + } + return k; +} + +/* +** Error returns from jsonLookupBlobStep() +*/ +#define JSON_BLOB_ERROR 0xffffffff +#define JSON_BLOB_NOTFOUND 0xfffffffe +#define JSON_BLOB_PATHERROR 0xfffffffd + +/* +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. +** +** Return JSON_BLOB_NOTFOUND if no such element exists. +*/ +static u32 jsonLookupBlobStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ + const char *zPath, /* The path to search */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, k, nKey, sz, n, iEnd; + const char *zKey; + u8 x; + + if( pParse->oom ) return 0; + if( zPath[0]==0 ) return iRoot; + if( zPath[0]=='.' ){ + x = pParse->aBlob[iRoot]; + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + testcase( nKey==0 ); + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + } + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; + iEnd = j+sz; + while( j<iEnd ){ + x = pParse->aBlob[j] & 0x0f; + if( x<JSONB_TEXT || x>JSONB_TEXTRAW ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + k = j+n; + if( k+sz>=iEnd ) return JSON_BLOB_ERROR; + if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( j+n+sz>iEnd ) return JSON_BLOB_ERROR; + return jsonLookupBlobStep(pParse, j, &zPath[i], pzErr); + } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + }else if( zPath[0]=='[' ){ + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; + } + if( i<2 || zPath[i]!=']' ){ + if( zPath[1]=='#' ){ + k = jsonbArrayCount(pParse, iRoot); + i = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int x = 0; + i = 3; + do{ + x = x*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( x>k ) return 0; + k -= x; + } + if( zPath[i]!=']' ){ + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + }else{ + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + } + j = iRoot+n; + iEnd = j+sz; + while( j<iEnd ){ + if( k==0 ){ + return jsonLookupBlobStep(pParse, j, &zPath[i+1], pzErr); + } + k--; + n = jsonbPayloadSize(pParse, j, &sz); + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + }else{ + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + return JSON_BLOB_NOTFOUND; +} + +/* +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. +*/ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob +){ + JsonParse x; + JsonString s; + + if( aBlob==0 ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonStringInit(&s, ctx); + jsonRenderBlob(&x, 0, &s); + jsonReturnString(&s); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx /* Return value for this function */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ) return; + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x = (char)pParse->aBlob[i+n]; + if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + else if( x=='+' && ALWAYS(sz>0) ){ n++; sz--; } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + if( bNeg ){ n--; sz++; } + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3_malloc( nOut+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(iIn=iOut=0; iIn<sz; iIn++){ + char c = z[iIn]; + if( c=='\\' ){ + c = z[++iIn]; + if( c=='u' ){ + u32 v = jsonHexToInt4(z+iIn+1); + iIn += 4; + if( v==0 ) break; + if( v<=0x7f ){ + zOut[iOut++] = (char)v; + }else if( v<=0x7ff ){ + zOut[iOut++] = (char)(0xc0 | (v>>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i<n-6 + && z[iIn+1]=='\\' + && z[iIn+2]=='u' + && ((vlo = jsonHexToInt4(z+iIn+3))&0xfc00)==0xdc00 + ){ + /* We have a surrogate pair */ + v = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + iIn += 6; + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); + iIn += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + iIn++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + iIn += 2; + continue; + }else{ + continue; + } + } /* end if( c=='\\' ) */ + zOut[iOut++] = c; + } /* end for() */ + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } + break; + } + } +} + +/* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. +*/ +static void jsonExtractFromBlob( + sqlite3_context *ctx, + sqlite3_value *pJson, + sqlite3_value *pPath, + int flags +){ + const char *zPath = (const char*)sqlite3_value_text(pPath); + const char *zErr = 0; + u32 i; + JsonParse px; + if( zPath==0 ) return; + if( zPath[0]=='$' ) zPath++; + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(pJson); + px.aBlob = (u8*)sqlite3_value_blob(pJson); + if( px.aBlob==0 ) return; + i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + if( i<px.nBlob ){ + jsonReturnFromBlob(&px, i, ctx); + }else if( i==JSON_BLOB_NOTFOUND ){ + return; /* Return NULL if not found */ + }else if( i==JSON_BLOB_ERROR ){ + sqlite3_result_error(ctx, "malformed JSONB", -1); + }else{ + sqlite3_result_error(ctx, zErr, -1); + } +} + /**************************************************************************** ** SQL functions used for testing and debugging @@ -2386,7 +4095,7 @@ static void jsonParseFunc( printf("iSubst = %u\n", p->iSubst); printf("iHold = %u\n", p->iHold); jsonDebugPrintNodeEntries(p->aNode, p->nNode); - jsonReturnJson(p, p->aNode, ctx, 1); + jsonReturnNodeAsJson(p, p->aNode, ctx, 1); } /* @@ -2402,14 +4111,84 @@ static void jsonTest1Func( UNUSED_PARAMETER(argc); sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } + +/* SQL Function: jsonb_test2(BLOB_JSON) +** +** Render BLOB_JSON back into text. +** Development testing only. +*/ +static void jsonbTest2( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + const u8 *aBlob; + int nBlob; + UNUSED_PARAMETER(argc); + + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + nBlob = sqlite3_value_bytes(argv[0]); + jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); +} #endif /* SQLITE_DEBUG */ /**************************************************************************** ** Scalar SQL function implementations ****************************************************************************/ +/* SQL Function: jsonb(JSON) +** +** Convert the input argument into JSONB (the SQLite binary encoding of +** JSON). +** +** If the input is TEXT, or NUMERIC, try to parse it as JSON. If the fails, +** raise an error. Otherwise, return the resulting BLOB value. +** +** If the input is a BLOB, check to see if the input is a plausible +** JSONB. If it is, return it unchanged. Raise an error if it is not. +** Note that there could be internal inconsistencies in the BLOB - this +** routine does not do a full byte-for-byte validity check of the +** JSON blob. +** +** If the input is NULL, return NULL. +*/ +static void jsonbFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *pParse; + int nJson; + const char *zJson; + JsonParse x; + UNUSED_PARAMETER(argc); + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* No-op */ + }else if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_value(ctx, argv[0]); + }else{ + zJson = (const char*)sqlite3_value_text(argv[0]); + if( zJson==0 ) return; + nJson = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.zJson = (char*)zJson; + x.nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } + jsonParseReset(pParse); + } +} + /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. @@ -2422,9 +4201,9 @@ static void jsonQuoteFunc( JsonString jx; UNUSED_PARAMETER(argc); - jsonInit(&jx, ctx); - jsonAppendValue(&jx, argv[0]); - jsonResult(&jx); + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2441,14 +4220,14 @@ static void jsonArrayFunc( int i; JsonString jx; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=0; i<argc; i++){ jsonAppendSeparator(&jx); - jsonAppendValue(&jx, argv[i]); + jsonAppendSqlValue(&jx, argv[i]); } jsonAppendChar(&jx, ']'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2499,15 +4278,6 @@ static void jsonArrayLengthFunc( } /* -** Bit values for the flags passed into jsonExtractFunc() or -** jsonSetFunc() via the user-data value. -*/ -#define JSON_JSON 0x01 /* Result is always JSON */ -#define JSON_SQL 0x02 /* Result is always SQL */ -#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ -#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ - -/* ** json_extract(JSON, PATH, ...) ** "->"(JSON,PATH) ** "->>"(JSON,PATH) @@ -2539,6 +4309,10 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_BLOB && argc==2 ){ + jsonExtractFromBlob(ctx, argv[0], argv[1], flags); + return; + } p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ @@ -2555,7 +4329,7 @@ static void jsonExtractFunc( ** LABEL ==> $.LABEL // PG compatible ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience */ - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); if( sqlite3Isdigit(zPath[0]) ){ jsonAppendRawNZ(&jx, "$[", 2); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); @@ -2566,27 +4340,27 @@ static void jsonExtractFunc( jsonAppendChar(&jx, 0); } pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); - jsonReset(&jx); + jsonStringReset(&jx); }else{ pNode = jsonLookup(p, zPath, 0, ctx); } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(p, pNode, ctx, 0); + jsonReturnNodeAsJson(p, pNode, ctx, 0); }else{ - jsonReturn(p, pNode, ctx); + jsonReturnNodeAsSql(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); + if( p->nErr==0 && pNode ) jsonReturnNodeAsSql(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each ** element of the array being the value selected by one of the PATHs */ int i; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=1; i<argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2594,17 +4368,17 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(p, pNode, &jx); + jsonRenderNodeAsText(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } } if( i==argc ){ jsonAppendChar(&jx, ']'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } - jsonReset(&jx); + jsonStringReset(&jx); } } @@ -2670,7 +4444,7 @@ static JsonNode *jsonMergePatch( pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; pParse->aNode[iRoot].u.iAppend = iStart; - VVA( pParse->aNode[iRoot].eU = 2 ); + JSON_VVA( pParse->aNode[iRoot].eU = 2 ); iRoot = iStart; pTarget = &pParse->aNode[iTarget]; } @@ -2706,7 +4480,7 @@ static void jsonPatchFunc( if( pResult && pX->oom==0 ){ jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); - jsonReturnJson(pX, pResult, ctx, 0); + jsonReturnNodeAsJson(pX, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } @@ -2733,12 +4507,12 @@ static void jsonObjectFunc( "of arguments", -1); return; } - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '{'); for(i=0; i<argc; i+=2){ if( sqlite3_value_type(argv[i])!=SQLITE_TEXT ){ sqlite3_result_error(ctx, "json_object() labels must be TEXT", -1); - jsonReset(&jx); + jsonStringReset(&jx); return; } jsonAppendSeparator(&jx); @@ -2746,10 +4520,10 @@ static void jsonObjectFunc( n = (u32)sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); - jsonAppendValue(&jx, argv[i+1]); + jsonAppendSqlValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2785,7 +4559,7 @@ static void jsonRemoveFunc( } } if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); } remove_done: jsonDebugPrintParse(p); @@ -2854,7 +4628,10 @@ static void jsonReplaceNode( k = jsonParseAddNode(p, JSON_STRING, n, zCopy); assert( k>0 || p->oom ); if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; - }else{ + break; + } + replace_with_json: + { JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); if( pPatch==0 ){ p->oom = 1; @@ -2871,9 +4648,13 @@ static void jsonReplaceNode( break; } default: { - jsonParseAddNode(p, JSON_NULL, 0, 0); - sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); - p->nErr++; + if( jsonFuncArgMightBeBinary(pValue) ){ + goto replace_with_json; + }else{ + jsonParseAddNode(p, JSON_NULL, 0, 0); + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + } break; } } @@ -2911,7 +4692,7 @@ static void jsonReplaceFunc( jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); replace_err: jsonDebugPrintParse(pParse); } @@ -2939,7 +4720,8 @@ static void jsonSetFunc( const char *zPath; u32 i; int bApnd; - int bIsSet = sqlite3_user_data(ctx)!=0; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; if( (argc&1)==0 ) { @@ -2963,7 +4745,7 @@ static void jsonSetFunc( } } jsonDebugPrintParse(pParse); - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); jsonSetDone: /* no cleanup required */; @@ -3101,24 +4883,34 @@ static void jsonArrayStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '['); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - jsonAppendValue(pStr, argv[0]); + jsonAppendSqlValue(pStr, argv[0]); } } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->bErr ){ if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -3207,7 +4999,7 @@ static void jsonObjectStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); @@ -3217,17 +5009,28 @@ static void jsonObjectStep( n = (u32)sqlite3_value_bytes(argv[0]); jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); + jsonAppendSqlValue(pStr, argv[1]); } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->bErr ){ if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -3381,7 +5184,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ if( pUp->eType==JSON_ARRAY ){ assert( pUp->eU==0 || pUp->eU==3 ); testcase( pUp->eU==3 ); - VVA( pUp->eU = 3 ); + JSON_VVA( pUp->eU = 3 ); if( iUp==p->i-1 ){ pUp->u.iKey = 0; }else{ @@ -3417,26 +5220,28 @@ static void jsonAppendObjectPathElement( JsonString *pStr, JsonNode *pNode ){ - int jj, nn; + int nn; const char *z; + int bNeedQuote = 0; assert( pNode->eType==JSON_STRING ); assert( pNode->jnFlags & JNODE_LABEL ); assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - if( (pNode->jnFlags & JNODE_RAW)==0 ){ - assert( nn>=2 ); - assert( z[0]=='"' || z[0]=='\'' ); - assert( z[nn-1]=='"' || z[0]=='\'' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){} - if( jj==nn-1 ){ - z++; - nn -= 2; - } - } + if( pNode->jnFlags & JNODE_RAW ){ + /* no-op */ + }else if( nn==0 || !sqlite3Isalpha(z[0]) ){ + bNeedQuote = 1; + }else{ + int jj; + for(jj=1; jj<nn && sqlite3Isalnum(z[jj]); jj++){} + bNeedQuote = jj<nn; + } + if( bNeedQuote ){ + jsonPrintf(nn+4, pStr, ".\"%.*s\"", nn, z); + }else{ + jsonPrintf(nn+2, pStr, ".%.*s", nn, z); } - jsonPrintf(nn+2, pStr, ".%.*s", nn, z); } /* Append the name of the path for element i to pStr @@ -3479,7 +5284,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -3495,7 +5300,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -3506,7 +5311,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); break; } case JEACH_ID: { @@ -3522,7 +5327,7 @@ static int jsonEachColumn( } case JEACH_FULLKEY: { JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); if( p->bRecursive ){ jsonEachComputePath(p, &x, p->i); }else{ @@ -3537,15 +5342,15 @@ static int jsonEachColumn( jsonAppendObjectPathElement(&x, pThis); } } - jsonResult(&x); + jsonReturnString(&x); break; } case JEACH_PATH: { if( p->bRecursive ){ JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); + jsonReturnString(&x); break; } /* For json_each() path and root are the same so fall through @@ -3559,8 +5364,12 @@ static int jsonEachColumn( break; } case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } break; } } @@ -3654,12 +5463,19 @@ static int jsonEachFilter( const char *z; const char *zRoot = 0; sqlite3_int64 n; + int isBinary; UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + z = (const char*)sqlite3_value_blob(argv[0]); + isBinary = 1; + }else{ + z = (const char*)sqlite3_value_text(argv[0]); + isBinary = 0; + } if( z==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; @@ -3669,9 +5485,11 @@ static int jsonEachFilter( n = sqlite3_value_bytes(argv[0]); p->sParse.zJson = sqlite3RCStrNew( n+1 ); if( p->sParse.zJson==0 ) return SQLITE_NOMEM; - memcpy(p->sParse.zJson, z, (size_t)n+1); + memcpy(p->sParse.zJson, z, (size_t)n+(isBinary==0)); + p->sParse.nJson = n; } p->sParse.bJsonIsRCStr = 1; + p->sParse.isBinary = isBinary; p->zJson = p->sParse.zJson; if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; @@ -3715,7 +5533,7 @@ static int jsonEachFilter( p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ assert( pNode->eU==0 ); - VVA( pNode->eU = 3 ); + JSON_VVA( pNode->eU = 3 ); pNode->u.iKey = 0; p->iEnd = p->i + pNode->n + 1; if( p->bRecursive ){ @@ -3797,33 +5615,49 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), + JFUNCTION(json, 1, 0, jsonRemoveFunc), + JFUNCTION(jsonb, 1, JSON_BLOB, jsonbFunc), + JFUNCTION(json_array, -1, 0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1, JSON_BLOB, jsonArrayFunc), + JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1, 0, jsonErrorFunc), + JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1, JSON_BLOB, jsonExtractFunc), + JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1, 0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1, JSON_BLOB, jsonSetFunc), + JFUNCTION(json_object, -1, 0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1, JSON_BLOB, jsonObjectFunc), + JFUNCTION(json_patch, 2, 0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2, JSON_BLOB, jsonPatchFunc), + JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), + JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1, JSON_BLOB, jsonRemoveFunc), + JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1, JSON_BLOB, jsonReplaceFunc), + JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), + JFUNCTION(jsonb_set, -1, JSON_ISSET|JSON_BLOB, jsonSetFunc), + JFUNCTION(json_type, 1, 0, jsonTypeFunc), + JFUNCTION(json_type, 2, 0, jsonTypeFunc), + JFUNCTION(json_valid, 1, 0, jsonValidFunc), #if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(json_parse, 1, 0, jsonParseFunc), + JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) }; sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); |