Add sqlite3.capi.sqlite3_js_posix_create_file() and oo1.OpfsDb.importDb() as alternatives for the newly-deprecated sqlite3_js_vfs_create_file().

FossilOrigin-Name: da6eaf8d8258f3e2c8633fd7faf4e90c3307b5c60bd8b69c626b3c82b19dbdef

4 files changed, 153 insertions, 23 deletions

diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js
index 572efeed5..60050461c 100644
--- a/ext/wasm/api/sqlite3-api-glue.js
+++ b/ext/wasm/api/sqlite3-api-glue.js
@@ -608,6 +608,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"],
     ["sqlite3_wasm_vfs_create_file", "int",
      "sqlite3_vfs*","string","*", "int"],
+    ["sqlite3_wasm_posix_create_file", "int", "string","*", "int"],
     ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"]
   ];
 
diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js
index fe167f025..ca5d1c44f 100644
--- a/ext/wasm/api/sqlite3-api-prologue.js
+++ b/ext/wasm/api/sqlite3-api-prologue.js
@@ -1357,11 +1357,73 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
   };
 
   /**
-     Achtung: this function does not work in debug builds of sqlite3
-     because its out-of-scope use of the sqlite3_vfs API triggers
-     unresolvable assertions in the core library.  That was
-     unfortunately not discovered until 2023-08-11. Because of that,
-     this function is now deprecated and should be used in new code.
+     If the current environment supports the POSIX file APIs, this routine
+     creates (or overwrites) the given file using those APIs. This is
+     primarily intended for use in Emscripten-based builds where the POSIX
+     APIs are transparently proxied by an in-memory virtual filesystem.
+     It may behave diffrently in other environments.
+
+     The first argument must be either a JS string or WASM C-string
+     holding the filename. Note that this routine does _not_ create
+     intermediary directories if the filename has a directory part.
+
+     The 2nd argument may either a valid WASM memory pointer, an
+     ArrayBuffer, or a Uint8Array. The 3rd must be the length, in
+     bytes, of the data array to copy. If the 2nd argument is an
+     ArrayBuffer or Uint8Array and the 3rd is not a positive integer
+     then the 3rd defaults to the array's byteLength value.
+
+     Results are undefined if data is a WASM pointer and dataLen is
+     exceeds data's bounds.
+
+     Throws if any arguments are invalid or if creating or writing to
+     the file fails.
+
+     Added in 3.43 as an alternative for the deprecated
+     sqlite3_js_vfs_create_file().
+  */
+  capi.sqlite3_js_posix_create_file = function(filename, data, dataLen){
+    let pData;
+    if(data && wasm.isPtr(data)){
+      pData = data;
+    }else if(data instanceof ArrayBuffer || data instanceof Uint8Array){
+      pData = wasm.allocFromTypedArray(data);
+      if(arguments.length<3 || !util.isInt32(dataLen) || dataLen<0){
+        dataLen = data.byteLength;
+      }
+    }else{
+      SQLite3Error.toss("Invalid 2nd argument for sqlite3_js_posix_create_file().");
+    }
+    try{
+      if(!util.isInt32(dataLen) || dataLen<0){
+        SQLite3Error.toss("Invalid 3rd argument for sqlite3_js_posix_create_file().");
+      }
+      const rc = wasm.sqlite3_wasm_posix_create_file(filename, pData, dataLen);
+      if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code",
+                               capi.sqlite3_js_rc_str(rc));
+    }finally{
+       wasm.dealloc(pData);
+    }
+  };
+
+  /**
+     Deprecation warning: this function does not work properly in
+     debug builds of sqlite3 because its out-of-scope use of the
+     sqlite3_vfs API triggers assertions in the core library.  That
+     was unfortunately not discovered until 2023-08-11. This function
+     is now deprecated and should not be used in new code.
+
+     Alternative options:
+
+     - "unix" VFS and its variants can get equivalent functionality
+       with sqlite3_js_posix_create_file().
+
+     - OPFS: use either sqlite3.oo1.OpfsDb.importDb(), for the "opfs"
+       VFS, or the importDb() method of the PoolUtil object provided
+       by the "opfs-sahpool" OPFS (noting that its VFS name may differ
+       depending on client-side configuration). We cannot proxy those
+       from here because the former is necessarily asynchronous and
+       the latter requires information not available to this function.
 
      Creates a file using the storage appropriate for the given
      sqlite3_vfs.  The first argument may be a VFS name (JS string
@@ -1408,9 +1470,13 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
        VFS nor the WASM environment imposes requirements which break it.
 
      - "opfs": uses OPFS storage and creates directory parts of the
-       filename.
+       filename. It can only be used to import an SQLite3 database
+       file and will fail if given anything else.
   */
   capi.sqlite3_js_vfs_create_file = function(vfs, filename, data, dataLen){
+    config.warn("sqlite3_js_vfs_create_file() is deprecated and",
+                "should be avoided because it can lead to C-level crashes.",
+                "See its documentation for alternative options.");
     let pData;
     if(data){
       if(wasm.isPtr(data)){
@@ -1438,7 +1504,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
       if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code",
                                capi.sqlite3_js_rc_str(rc));
     }finally{
-      wasm.dealloc(pData);
+       wasm.dealloc(pData);
     }
   };
 
diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
index 58c511082..93482505a 100644
--- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
@@ -1168,11 +1168,41 @@ const installOpfsVfs = function callee(options){
       doDir(opt.directory, 0);
     };
 
-    //TODO to support fiddle and worker1 db upload:
-    //opfsUtil.createFile = function(absName, content=undefined){...}
-    //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this
-    //purpose but its interface and name are still under
-    //consideration.
+    /**
+       Asynchronously imports the given bytes (a byte array or
+       ArrayBuffer) into the given database file.
+
+       It very specifically requires the input to be an SQLite3
+       database and throws if that's not the case.  It does so in
+       order to prevent this function from taking on a larger scope
+       than it is specifically intended to. i.e. we do not want it to
+       become a convenience for importing arbitrary files into OPFS.
+
+       Throws on error. Resolves to the number of bytes written.
+    */
+    opfsUtil.importDb = async function(filename, bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const n = bytes.byteLength;
+      if(n<512 || n%512!=0){
+        toss("Byte array size is invalid for an SQLite db.");
+      }
+      const header = "SQLite format 3";
+      for(let i = 0; i < header.length; ++i){
+        if( header.charCodeAt(i) !== bytes[i] ){
+          toss("Input does not contain an SQLite database header.");
+        }
+      }
+      const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
+      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+      const sah = await hFile.createSyncAccessHandle();
+      sah.truncate(0);
+      const nWrote = sah.write(bytes, {at: 0});
+      sah.close();
+      if(nWrote != n){
+        toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+      }
+      return nWrote;
+    };
 
     if(sqlite3.oo1){
       const OpfsDb = function(...args){
@@ -1182,6 +1212,7 @@ const installOpfsVfs = function callee(options){
       };
       OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
       sqlite3.oo1.OpfsDb = OpfsDb;
+      OpfsDb.importDb = opfsUtil.importDb;
       sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql(
         opfsVfs.pointer,
         function(oo1Db, sqlite3){
diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c
index 431eddceb..fcfbc0692 100644
--- a/ext/wasm/api/sqlite3-wasm.c
+++ b/ext/wasm/api/sqlite3-wasm.c
@@ -141,9 +141,6 @@
 #ifndef SQLITE_OMIT_UTF16
 # define SQLITE_OMIT_UTF16 1
 #endif
-#ifndef SQLITE_OMIT_WAL
-# define SQLITE_OMIT_WAL 1
-#endif
 #ifndef SQLITE_OS_KV_OPTIONAL
 # define SQLITE_OS_KV_OPTIONAL 1
 #endif
@@ -1353,6 +1350,13 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema,
 ** This function is NOT part of the sqlite3 public API. It is strictly
 ** for use by the sqlite project's own JS/WASM bindings.
 **
+** ACHTUNG: it was discovered on 2023-08-11 that, with SQLITE_DEBUG,
+** this function's out-of-scope use of the sqlite3_vfs/file/io_methods
+** APIs leads to triggering of assertions in the core library. Its use
+** is now deprecated and VFS-specific APIs for importing files need to
+** be found to replace it. sqlite3_wasm_posix_create_file() is
+** suitable for the "unix" family of VFSes.
+**
 ** Creates a new file using the I/O API of the given VFS, containing
 ** the given number of bytes of the given data. If the file exists, it
 ** is truncated to the given length and populated with the given
@@ -1395,16 +1399,17 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs,
                                   const char *zFilename,
                                   const unsigned char * pData,
                                   int nData ){
-#ifdef SQLITE_DEBUG
-  fprintf(stderr,"%s does not work in debug builds because its out-of-scope use of "
-          "the sqlite3_vfs API triggers assertions in the core library.\n", __func__);
-  /* ^^^ That was unfortunately not discovered until 2023-08-11. */
-  return SQLITE_ERROR;
-#else
   int rc;
   sqlite3_file *pFile = 0;
   sqlite3_io_methods const *pIo;
-  const int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+  const int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
+#if 0 && defined(SQLITE_DEBUG)
+    | SQLITE_OPEN_MAIN_JOURNAL
+    /* ^^^^ This is for testing a horrible workaround to avoid
+       triggering a specific assert() in os_unix.c:unixOpen(). Please
+       do not enable this in real builds. */
+#endif
+    ;
   int flagsOut = 0;
   int fileExisted = 0;
   int doUnlock = 0;
@@ -1468,7 +1473,34 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs,
   RC;
 #undef RC
   return rc;
-#endif
+}
+
+/**
+** This function is NOT part of the sqlite3 public API. It is strictly
+** for use by the sqlite project's own JS/WASM bindings.
+**
+** Creates or overwrites a file using the POSIX file API,
+** i.e. Emscripten's virtual filesystem. Creates or truncates
+** zFilename, appends pData bytes to it, and returns 0 on success or
+** SQLITE_IOERR on error.
+*/
+SQLITE_WASM_EXPORT
+int sqlite3_wasm_posix_create_file( const char *zFilename,
+                                    const unsigned char * pData,
+                                    int nData ){
+  int rc;
+  FILE * pFile = 0;
+  int fileExisted = 0;
+  size_t nWrote = 1;
+
+  if( !zFilename || nData<0 || (pData==0 && nData>0) ) return SQLITE_MISUSE;
+  pFile = fopen(zFilename, "w");
+  if( 0==pFile ) return SQLITE_IOERR;
+  if( nData>0 ){
+    nWrote = fwrite(pData, (size_t)nData, 1, pFile);
+  }
+  fclose(pFile);
+  return 1==nWrote ? 0 : SQLITE_IOERR;
 }
 
 /*