Extend the importDb() method of both OPFS VFSes to (A) support reading in an async streaming fashion via a callback and (B) automatically disable WAL mode in the imported db.

FossilOrigin-Name: 9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2

3 files changed, 171 insertions, 24 deletions

diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js
index ca5d1c44f..5abb13b99 100644
--- a/ext/wasm/api/sqlite3-api-prologue.js
+++ b/ext/wasm/api/sqlite3-api-prologue.js
@@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     isSharedTypedArray,
     toss: function(...args){throw new Error(args.join(' '))},
     toss3,
-    typedArrayPart
-  };
+    typedArrayPart,
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       lead bytes of that buffer do not hold a SQLite3 database header,
+       else it returns without side effects.
+
+       Added in 3.44.
+    */
+    affirmDbHeader: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const header = "SQLite format 3";
+      if( header.length > bytes.byteLength ){
+        toss3("Input does not contain an SQLite3 database header.");
+      }
+      for(let i = 0; i < header.length; ++i){
+        if( header.charCodeAt(i) !== bytes[i] ){
+          toss3("Input does not contain an SQLite3 database header.");
+        }
+      }
+    },
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       database does not, at a cursory glance, appear to be an SQLite3
+       database. It only examines the size and header, but further
+       checks may be added in the future.
+
+       Added in 3.44.
+    */
+    affirmIsDb: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const n = bytes.byteLength;
+      if(n<512 || n%512!==0) {
+        toss3("Byte array size",n,"is invalid for an SQLite3 db.");
+      }
+      util.affirmDbHeader(bytes);
+    }
+  }/*util*/;
 
   Object.assign(wasm, {
     /**
diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
index 709d3414c..8e874f728 100644
--- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
@@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
   const toss3 = sqlite3.util.toss3;
   const initPromises = Object.create(null);
   const capi = sqlite3.capi;
+  const util = sqlite3.util;
   const wasm = sqlite3.wasm;
   // Config opts for the VFS...
   const SECTOR_SIZE = 4096;
@@ -869,9 +870,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
       return b;
     }
 
+    //! Impl for importDb() when its 2nd arg is a function.
+    async importDbChunked(name, callback){
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
+      sah.truncate(0);
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at:  HEADER_OFFSET_DATA + nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array(2), {
+          at: HEADER_OFFSET_DATA + 18
+        }/*force db out of WAL mode*/);
+      }catch(e){
+        this.setAssociatedPath(sah, '', 0);
+      }
+      this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
+      return nWrote;
+    }
+
     //! Documented elsewhere in this file.
     importDb(name, bytes){
-      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
+      else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
       const n = bytes.byteLength;
       if(n<512 || n%512!=0){
         toss("Byte array size is invalid for an SQLite db.");
@@ -882,16 +922,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           toss("Input does not contain an SQLite database header.");
         }
       }
-      const sah = this.#mapFilenameToSAH.get(name)
-            || this.nextAvailableSAH()
-            || toss("No available handles to import to.");
       const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
       if(nWrote != n){
         this.setAssociatedPath(sah, '', 0);
         toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
       }else{
+        sah.write(new Uint8Array([0,0]), {at: HEADER_OFFSET_DATA+18}
+                   /* force db out of WAL mode */);
         this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
       }
+      return nWrote;
     }
 
   }/*class OpfsSAHPool*/;
@@ -1098,6 +1138,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      automatically clean up any non-database files so importing them
      is pointless.
 
+     If passed a function for its second argument, its behavior
+     changes to asynchronous and it imports its data in chunks fed to
+     it by the given callback function. It calls the callback (which
+     may be async) repeatedly, expecting either a Uint8Array or
+     ArrayBuffer (to denote new input) or undefined (to denote
+     EOF). For so long as the callback continues to return
+     non-undefined, it will append incoming data to the given
+     VFS-hosted database file. The result of the resolved Promise when
+     called this way is the size of the resulting database.
+
+     On succes this routine rewrites the database header bytes in the
+     output file (not the input array) to force disabling of WAL mode.
+
      On a write error, the handle is removed from the pool and made
      available for re-use.
 
diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
index 93482505a..5edb129ea 100644
--- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
@@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
     const error =  (...args)=>logImpl(0, ...args);
     const toss = sqlite3.util.toss;
     const capi = sqlite3.capi;
+    const util = sqlite3.util;
     const wasm = sqlite3.wasm;
     const sqlite3_vfs = capi.sqlite3_vfs;
     const sqlite3_file = capi.sqlite3_file;
@@ -1169,39 +1170,97 @@ const installOpfsVfs = function callee(options){
     };
 
     /**
+       impl of importDb() when it's given a function as its second
+       argument.
+    */
+    const importDbChunked = async function(filename, callback){
+      const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
+      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+      const sah = await hFile.createSyncAccessHandle();
+      sah.truncate(0);
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at: nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array(2), {at: 18}/*force db out of WAL mode*/);
+        return nWrote;
+      }catch(e){
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally {
+        await sah.close();
+      }
+    };
+
+    /**
        Asynchronously imports the given bytes (a byte array or
        ArrayBuffer) into the given database file.
 
+       If passed a function for its second argument, its behaviour
+       changes to async and it imports its data in chunks fed to it by
+       the given callback function. It calls the callback (which may
+       be async) repeatedly, expecting either a Uint8Array or
+       ArrayBuffer (to denote new input) or undefined (to denote
+       EOF). For so long as the callback continues to return
+       non-undefined, it will append incoming data to the given
+       VFS-hosted database file. When called this way, the resolved
+       value of the returned Promise is the number of bytes written to
+       the target file.
+
        It very specifically requires the input to be an SQLite3
        database and throws if that's not the case.  It does so in
        order to prevent this function from taking on a larger scope
        than it is specifically intended to. i.e. we do not want it to
        become a convenience for importing arbitrary files into OPFS.
 
-       Throws on error. Resolves to the number of bytes written.
+       This routine rewrites the database header bytes in the output
+       file (not the input array) to force disabling of WAL mode.
+
+       On error this throws and the state of the input file is
+       undefined (it depends on where the exception was triggered).
+
+       On success, resolves to the number of bytes written.
     */
     opfsUtil.importDb = async function(filename, bytes){
+      if( bytes instanceof Function ){
+        return importDbChunked(filename, bytes);
+      }
       if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      util.affirmIsDb(bytes);
       const n = bytes.byteLength;
-      if(n<512 || n%512!=0){
-        toss("Byte array size is invalid for an SQLite db.");
-      }
-      const header = "SQLite format 3";
-      for(let i = 0; i < header.length; ++i){
-        if( header.charCodeAt(i) !== bytes[i] ){
-          toss("Input does not contain an SQLite database header.");
-        }
-      }
       const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
-      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
-      const sah = await hFile.createSyncAccessHandle();
-      sah.truncate(0);
-      const nWrote = sah.write(bytes, {at: 0});
-      sah.close();
-      if(nWrote != n){
-        toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+      let sah, err, nWrote = 0;
+      try {
+        const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+        sah = await hFile.createSyncAccessHandle();
+        sah.truncate(0);
+        nWrote = sah.write(bytes, {at: 0});
+        if(nWrote != n){
+          toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+        }
+        sah.write(new Uint8Array(2), {at: 18}) /* force db out of WAL mode */;
+        return nWrote;
+      }catch(e){
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally{
+        if( sah ) await sah.close();
       }
-      return nWrote;
     };
 
     if(sqlite3.oo1){