Add the SQLITE_RESULT_SUBTYPE flag for application-defined functions. Add

the -DSQLITE_STRICT_SUBTYPE=1 compile-time option that raises an error if
any function invokes sqlite3_result_subtype() without the SQLITE_RESULT_SUBTYPE
flag.  SQLITE_RESULT_SUBTYPE prevents an indexed value of that function from
being used to replace an equivalent expression, since the indexed expression
does not carry the subtype.  Fix for the problem described at
[forum:/forumpost/68d284c86b082c3e|forum post 68d284c86b082c3e].

FossilOrigin-Name: ba789a7804ab96d81b15d6ef6fed1f802fa69db47cf91d368933e55289fa1d6e

7 files changed, 122 insertions, 52 deletions

diff --git a/src/json.c b/src/json.c
index 91b96df37..0f11ace02 100644
--- a/src/json.c
+++ b/src/json.c
@@ -839,7 +839,8 @@ static void jsonReturnJson(
   JsonParse *pParse,          /* The complete JSON */
   JsonNode *pNode,            /* Node to return */
   sqlite3_context *pCtx,      /* Return value for this function */
-  int bGenerateAlt            /* Also store the rendered text in zAlt */
+  int bGenerateAlt,           /* Also store the rendered text in zAlt */
+  int omitSubtype             /* Do not call sqlite3_result_subtype() */
 ){
   JsonString s;
   if( pParse->oom ){
@@ -854,7 +855,7 @@ static void jsonReturnJson(
       pParse->nAlt = s.nUsed;
     }
     jsonResult(&s);
-    sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
+    if( !omitSubtype ) sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
   }
 }
 
@@ -895,7 +896,8 @@ static u32 jsonHexToInt4(const char *z){
 static void jsonReturn(
   JsonParse *pParse,          /* Complete JSON parse tree */
   JsonNode *pNode,            /* Node to return */
-  sqlite3_context *pCtx       /* Return value for this function */
+  sqlite3_context *pCtx,      /* Return value for this function */
+  int omitSubtype             /* Do not call sqlite3_result_subtype() */
 ){
   switch( pNode->eType ){
     default: {
@@ -1041,7 +1043,7 @@ static void jsonReturn(
     }
     case JSON_ARRAY:
     case JSON_OBJECT: {
-      jsonReturnJson(pParse, pNode, pCtx, 0);
+      jsonReturnJson(pParse, pNode, pCtx, 0, omitSubtype);
       break;
     }
   }
@@ -2393,7 +2395,7 @@ static void jsonParseFunc(
   printf("iSubst    = %u\n", p->iSubst);
   printf("iHold     = %u\n", p->iHold);
   jsonDebugPrintNodeEntries(p->aNode, p->nNode);
-  jsonReturnJson(p, p->aNode, ctx, 1);
+  jsonReturnJson(p, p->aNode, ctx, 1, 0);
 }
 
 /*
@@ -2579,15 +2581,14 @@ static void jsonExtractFunc(
       }
       if( pNode ){
         if( flags & JSON_JSON ){
-          jsonReturnJson(p, pNode, ctx, 0);
+          jsonReturnJson(p, pNode, ctx, 0, 0);
         }else{
-          jsonReturn(p, pNode, ctx);
-          sqlite3_result_subtype(ctx, 0);
+          jsonReturn(p, pNode, ctx, 1);
         }
       }
     }else{
       pNode = jsonLookup(p, zPath, 0, ctx);
-      if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx);
+      if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx, 0);
     }
   }else{
     /* Two or more PATH arguments results in a JSON array with each
@@ -2713,7 +2714,7 @@ static void jsonPatchFunc(
   if( pResult && pX->oom==0 ){
     jsonDebugPrintParse(pX);
     jsonDebugPrintNode(pResult);
-    jsonReturnJson(pX, pResult, ctx, 0);
+    jsonReturnJson(pX, pResult, ctx, 0, 0);
   }else{
     sqlite3_result_error_nomem(ctx);
   }
@@ -2792,7 +2793,7 @@ static void jsonRemoveFunc(
     }
   }
   if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){
-    jsonReturnJson(pParse, pParse->aNode, ctx, 1);
+    jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
   }
 remove_done:
   jsonDebugPrintParse(p);
@@ -2921,7 +2922,7 @@ static void jsonReplaceFunc(
       jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]);
     }
   }
-  jsonReturnJson(pParse, pParse->aNode, ctx, 1);
+  jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
 replace_err:
   jsonDebugPrintParse(pParse);
   jsonParseFree(pParse);
@@ -2975,7 +2976,7 @@ static void jsonSetFunc(
     }
   }
   jsonDebugPrintParse(pParse);
-  jsonReturnJson(pParse, pParse->aNode, ctx, 1);
+  jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
 jsonSetDone:
   jsonParseFree(pParse);
 }
@@ -3490,7 +3491,7 @@ static int jsonEachColumn(
     case JEACH_KEY: {
       if( p->i==0 ) break;
       if( p->eType==JSON_OBJECT ){
-        jsonReturn(&p->sParse, pThis, ctx);
+        jsonReturn(&p->sParse, pThis, ctx, 0);
       }else if( p->eType==JSON_ARRAY ){
         u32 iKey;
         if( p->bRecursive ){
@@ -3506,7 +3507,7 @@ static int jsonEachColumn(
     }
     case JEACH_VALUE: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
-      jsonReturn(&p->sParse, pThis, ctx);
+      jsonReturn(&p->sParse, pThis, ctx, 0);
       break;
     }
     case JEACH_TYPE: {
@@ -3517,7 +3518,7 @@ static int jsonEachColumn(
     case JEACH_ATOM: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
       if( pThis->eType>=JSON_ARRAY ) break;
-      jsonReturn(&p->sParse, pThis, ctx);
+      jsonReturn(&p->sParse, pThis, ctx, 0);
       break;
     }
     case JEACH_ID: {
@@ -3810,37 +3811,43 @@ static sqlite3_module jsonTreeModule = {
 void sqlite3RegisterJsonFunctions(void){
 #ifndef SQLITE_OMIT_JSON
   static FuncDef aJsonFunc[] = {
-    /*              Uses cache ------,  ,---- Might return JSON (subtype J) */
-    /*          Num args ________    |  |    ___ Flags                      */
-    /*                           \   |  |   /                               */
-    JFUNCTION(json,               1, 1, 1, 0,          jsonRemoveFunc),
-    JFUNCTION(json_array,        -1, 0, 1, 0,          jsonArrayFunc),
-    JFUNCTION(json_array_length,  1, 1, 0, 0,          jsonArrayLengthFunc),
-    JFUNCTION(json_array_length,  2, 1, 0, 0,          jsonArrayLengthFunc),
-    JFUNCTION(json_error_position,1, 1, 0, 0,          jsonErrorFunc),
-    JFUNCTION(json_extract,      -1, 1, 1, 0,          jsonExtractFunc),
-    JFUNCTION(->,                 2, 1, 1, JSON_JSON,  jsonExtractFunc),
-    JFUNCTION(->>,                2, 1, 0, JSON_SQL,   jsonExtractFunc),
-    JFUNCTION(json_insert,       -1, 1, 1, 0,          jsonSetFunc),
-    JFUNCTION(json_object,       -1, 0, 1, 0,          jsonObjectFunc),
-    JFUNCTION(json_patch,         2, 1, 1, 0,          jsonPatchFunc),
-    JFUNCTION(json_quote,         1, 0, 1, 0,          jsonQuoteFunc),
-    JFUNCTION(json_remove,       -1, 1, 1, 0,          jsonRemoveFunc),
-    JFUNCTION(json_replace,      -1, 1, 1, 0,          jsonReplaceFunc),
-    JFUNCTION(json_set,          -1, 1, 1, JSON_ISSET, jsonSetFunc),
-    JFUNCTION(json_type,          1, 1, 0, 0,          jsonTypeFunc),
-    JFUNCTION(json_type,          2, 1, 0, 0,          jsonTypeFunc),
-    JFUNCTION(json_valid,         1, 1, 0, 0,          jsonValidFunc),
+    /*                     calls sqlite3_result_subtype()                    */
+    /*                                  |                                    */
+    /*              Uses cache ______   |   __ calls sqlite3_value_subtype() */
+    /*                               |  |  |                                 */
+    /*          Num args _________   |  |  |   ___ Flags                     */
+    /*                            |  |  |  |  |                              */
+    /*                            |  |  |  |  |                              */
+    JFUNCTION(json,               1, 1, 1, 0, 0,          jsonRemoveFunc),
+    JFUNCTION(json_array,        -1, 0, 1, 1, 0,          jsonArrayFunc),
+    JFUNCTION(json_array_length,  1, 1, 0, 0, 0,          jsonArrayLengthFunc),
+    JFUNCTION(json_array_length,  2, 1, 0, 0, 0,          jsonArrayLengthFunc),
+    JFUNCTION(json_error_position,1, 1, 0, 0, 0,          jsonErrorFunc),
+    JFUNCTION(json_extract,      -1, 1, 1, 0, 0,          jsonExtractFunc),
+    JFUNCTION(->,                 2, 1, 1, 0, JSON_JSON,  jsonExtractFunc),
+    JFUNCTION(->>,                2, 1, 0, 0, JSON_SQL,   jsonExtractFunc),
+    JFUNCTION(json_insert,       -1, 1, 1, 1, 0,          jsonSetFunc),
+    JFUNCTION(json_object,       -1, 0, 1, 1, 0,          jsonObjectFunc),
+    JFUNCTION(json_patch,         2, 1, 1, 0, 0,          jsonPatchFunc),
+    JFUNCTION(json_quote,         1, 0, 1, 1, 0,          jsonQuoteFunc),
+    JFUNCTION(json_remove,       -1, 1, 1, 0, 0,          jsonRemoveFunc),
+    JFUNCTION(json_replace,      -1, 1, 1, 1, 0,          jsonReplaceFunc),
+    JFUNCTION(json_set,          -1, 1, 1, 1, JSON_ISSET, jsonSetFunc),
+    JFUNCTION(json_type,          1, 1, 0, 0, 0,          jsonTypeFunc),
+    JFUNCTION(json_type,          2, 1, 0, 0, 0,          jsonTypeFunc),
+    JFUNCTION(json_valid,         1, 1, 0, 0, 0,          jsonValidFunc),
 #if SQLITE_DEBUG
-    JFUNCTION(json_parse,         1, 1, 0, 0,          jsonParseFunc),
-    JFUNCTION(json_test1,         1, 1, 0, 0,          jsonTest1Func),
+    JFUNCTION(json_parse,         1, 1, 1, 0, 0,          jsonParseFunc),
+    JFUNCTION(json_test1,         1, 1, 0, 1, 0,          jsonTest1Func),
 #endif
     WAGGREGATE(json_group_array,  1, 0, 0,
        jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse,
-       SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC),
+       SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|
+       SQLITE_DETERMINISTIC),
     WAGGREGATE(json_group_object, 2, 0, 0,
        jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse,
-       SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC)
+       SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|
+       SQLITE_DETERMINISTIC)
   };
   sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc));
 #endif
diff --git a/src/main.c b/src/main.c
index fbe00f5fa..6acfdc325 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1914,7 +1914,7 @@ int sqlite3CreateFunc(
   assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC );
   assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY );
   extraFlags = enc &  (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY|
-                       SQLITE_SUBTYPE|SQLITE_INNOCUOUS);
+                       SQLITE_SUBTYPE|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE);
   enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY);
 
   /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE.  But
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 2317d98f7..53c037c3b 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -5573,13 +5573,27 @@ int sqlite3_create_window_function(
 ** </dd>
 **
 ** [[SQLITE_SUBTYPE]] <dt>SQLITE_SUBTYPE</dt><dd>
-** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call
+** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call
 ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments.
-** Specifying this flag makes no difference for scalar or aggregate user
-** functions. However, if it is not specified for a user-defined window
-** function, then any sub-types belonging to arguments passed to the window
-** function may be discarded before the window function is called (i.e.
-** sqlite3_value_subtype() will always return 0).
+** This flag instructs SQLite to omit some corner-case optimizations that
+** might disrupt the operation of the [sqlite3_value_subtype()] function,
+** causing it to return zero rather than the correct subtype().
+** SQL functions that invokes [sqlite3_value_subtype()] should have this
+** property.  If the SQLITE_SUBTYPE property is omitted, then the return
+** value from [sqlite3_value_subtype()] might sometimes be zero even though
+** a non-zero subtype was specified by the function argument expression.
+**
+** [[SQLITE_RESULT_SUBTYPE]] <dt>SQLITE_RESULT_SUBTYPE</dt><dd>
+** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call
+** [sqlite3_result_subtype()] to cause a sub-type to be associated with its
+** result.
+** Every function that invokes [sqlite3_result_subtype()] should have this
+** property.  If it does not, then the call to [sqlite3_result_subtype()]
+** might become a no-op if the function is used as term in an
+** [expression index].  On the other hand, SQL functions that never invoke
+** [sqlite3_result_subtype()] should avoid setting this property, as the
+** purpose of this property is to disable certain optimizations that are
+** incompatible with subtypes.
 ** </dd>
 ** </dl>
 */
@@ -5587,6 +5601,7 @@ int sqlite3_create_window_function(
 #define SQLITE_DIRECTONLY       0x000080000
 #define SQLITE_SUBTYPE          0x000100000
 #define SQLITE_INNOCUOUS        0x000200000
+#define SQLITE_RESULT_SUBTYPE   0x001000000
 
 /*
 ** CAPI3REF: Deprecated Functions
@@ -5783,6 +5798,12 @@ int sqlite3_value_encoding(sqlite3_value*);
 ** information can be used to pass a limited amount of context from
 ** one SQL function to another.  Use the [sqlite3_result_subtype()]
 ** routine to set the subtype for the return value of an SQL function.
+**
+** Every [application-defined SQL function] that invoke this interface
+** should include the [SQLITE_SUBTYPE] property in the text
+** encoding argument when the function is [sqlite3_create_function|registered].
+** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype()
+** might return zero instead of the upstream subtype in some corner cases.
 */
 unsigned int sqlite3_value_subtype(sqlite3_value*);
 
@@ -6202,6 +6223,20 @@ int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n);
 ** higher order bits are discarded.
 ** The number of subtype bytes preserved by SQLite might increase
 ** in future releases of SQLite.
+**
+** Every [application-defined SQL function] that invokes this interface
+** should include the [SQLITE_RESULT_SUBTYPE] property in its
+** text encoding argument when the SQL function is
+** [sqlite3_create_function|registered].  If the [SQLITE_RESULT_SUBTYPE]
+** property is omitted from the function that invokes sqlite3_result_subtype(),
+** then in some cases the sqlite3_result_subtype() might fail to set
+** the result subtype.
+**
+** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any
+** SQL function that invokes the sqlite3_result_subtype() interface
+** and that does not have the SQLITE_RESULT_SUBTYPE property will raise
+** an error.  Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1
+** by default.
 */
 void sqlite3_result_subtype(sqlite3_context*,unsigned int);
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 20e23ed2b..bb61cb691 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2018,10 +2018,11 @@ struct FuncDestructor {
 #define SQLITE_FUNC_WINDOW   0x00010000 /* Built-in window-only function */
 #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */
 #define SQLITE_FUNC_DIRECT   0x00080000 /* Not for use in TRIGGERs or VIEWs */
-#define SQLITE_FUNC_SUBTYPE  0x00100000 /* Result likely to have sub-type */
+/* SQLITE_SUBTYPE            0x00100000 // Consumer of subtypes */
 #define SQLITE_FUNC_UNSAFE   0x00200000 /* Function has side effects */
 #define SQLITE_FUNC_INLINE   0x00400000 /* Functions implemented in-line */
 #define SQLITE_FUNC_BUILTIN  0x00800000 /* This is a built-in function */
+/*  SQLITE_RESULT_SUBTYPE    0x01000000 // Generator of subtypes */
 #define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */
 
 /* Identifier numbers for each in-line function */
@@ -2113,9 +2114,10 @@ struct FuncDestructor {
 #define MFUNCTION(zName, nArg, xPtr, xFunc) \
   {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
    xPtr, 0, xFunc, 0, 0, 0, #zName, {0} }
-#define JFUNCTION(zName, nArg, bUseCache, bSubtype, iArg, xFunc) \
+#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, iArg, xFunc) \
   {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\
-   SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|((bSubtype)*SQLITE_SUBTYPE), \
+   SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\
+   ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \
    SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
 #define INLINE_FUNC(zName, nArg, iArg, mFlags) \
   {nArg, SQLITE_FUNC_BUILTIN|\
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 6724035fd..570cb3d8b 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -539,6 +539,18 @@ void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( pCtx==0 ) return;
 #endif
+#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0
+  if( pCtx->pFunc!=0
+   && (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0
+  ){
+    char zErr[200];
+    sqlite3_snprintf(sizeof(zErr), zErr,
+                     "misuse of sqlite3_result_subtype() by %s()", 
+                     pCtx->pFunc->zName);
+    sqlite3_result_error(pCtx, zErr, -1);
+    return;
+  }
+#endif /* SQLITE_STRICT_SUBTYPE */
   pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pOut->db->mutex) );
   pOut->eSubtype = eSubtype & 0xff;
diff --git a/src/where.c b/src/where.c
index 05ae24f7b..555bd6a34 100644
--- a/src/where.c
+++ b/src/where.c
@@ -5810,6 +5810,20 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
       continue;
     }
     if( sqlite3ExprIsConstant(pExpr) ) continue;
+    if( pExpr->op==TK_FUNCTION ){
+      /* Functions that might set a subtype should not be replaced by the
+      ** value taken from an expression index since the index omits the
+      ** subtype.  https://sqlite.org/forum/forumpost/68d284c86b082c3e */
+      int n;
+      FuncDef *pDef;
+      sqlite3 *db = pParse->db;
+      assert( ExprUseXList(pExpr) );
+      n = ALWAYS(pExpr->x.pList) ? pExpr->x.pList->nExpr : 0;
+      pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
+      if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
+        continue;
+      }
+    }
     p = sqlite3DbMallocRaw(pParse->db,  sizeof(IndexedExpr));
     if( p==0 ) break;
     p->pIENext = pParse->pIdxEpr;
diff --git a/src/window.c b/src/window.c
index 2c449592d..62df349fb 100644
--- a/src/window.c
+++ b/src/window.c
@@ -1038,7 +1038,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
       assert( ExprUseXList(pWin->pOwner) );
       assert( pWin->pWFunc!=0 );
       pArgs = pWin->pOwner->x.pList;
-      if( pWin->pWFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){
+      if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){
         selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist);
         pWin->iArgCol = (pSublist ? pSublist->nExpr : 0);
         pWin->bExprArgs = 1;