Refactor a significant chunk of the OPFS sqlite3_vfs init code into sqlite3.VfsHelper, and internal-use-only API encapsulating code relevant to creating new VFSes in JS. Intended to assist in pending experimentation with an alternative OPFS VFS.

FossilOrigin-Name: e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8

6 files changed, 262 insertions, 113 deletions

diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md
index d33c4a583..37d986584 100644
--- a/ext/wasm/api/README.md
+++ b/ext/wasm/api/README.md
@@ -23,13 +23,13 @@ The overall idea is that the following files get concatenated
 together, in the listed order, the resulting file is loaded by a
 browser client:
 
-- `sqlite3-api-prologue.js`\  
+- **`sqlite3-api-prologue.js`**\  
   Contains the initial bootstrap setup of the sqlite3 API
   objects. This is exposed as a function, rather than objects, so that
   the next step can pass in a config object which abstracts away parts
   of the WASM environment, to facilitate plugging it in to arbitrary
   WASM toolchains.
-- `../common/whwasmutil.js`\  
+- **`../common/whwasmutil.js`**\  
   A semi-third-party collection of JS/WASM utility code intended to
   replace much of the Emscripten glue. The sqlite3 APIs internally use
   these APIs instead of their Emscripten counterparts, in order to be
@@ -38,47 +38,52 @@ browser client:
   toolchains. It is "semi-third-party" in that it was created in order
   to support this tree but is standalone and maintained together
   with...
-- `../jaccwabyt/jaccwabyt.js`\  
+- **`../jaccwabyt/jaccwabyt.js`**\  
   Another semi-third-party API which creates bindings between JS
   and C structs, such that changes to the struct state from either JS
   or C are visible to the other end of the connection. This is also an
   independent spinoff project, conceived for the sqlite3 project but
   maintained separately.
-- `sqlite3-api-glue.js`\  
+- **`sqlite3-api-glue.js`**\  
   Invokes functionality exposed by the previous two files to
   flesh out low-level parts of `sqlite3-api-prologue.js`. Most of
   these pieces related to the `sqlite3.capi.wasm` object.
-- `sqlite3-api-build-version.js`\  
+- **`sqlite3-api-build-version.js`**\  
   Gets created by the build process and populates the
   `sqlite3.version` object. This part is not critical, but records the
   version of the library against which this module was built.
-- `sqlite3-api-oo1.js`\  
+- **`sqlite3-vfs-helper.js`**\  
+  This internal-use-only file installs `sqlite3.VfsHelper` for use by
+  `sqlite3-api-*.js` files which create `sqlite3_vfs` implemenations.
+  `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after
+  the library is finished initializing.
+- **`sqlite3-api-oo1.js`**\  
   Provides a high-level object-oriented wrapper to the lower-level C
   API, colloquially known as OO API #1. Its API is similar to other
   high-level sqlite3 JS wrappers and should feel relatively familiar
   to anyone familiar with such APIs. That said, it is not a "required
   component" and can be elided from builds which do not want it.
-- `sqlite3-api-worker1.js`\  
+- **`sqlite3-api-worker1.js`**\  
   A Worker-thread-based API which uses OO API #1 to provide an
   interface to a database which can be driven from the main Window
   thread via the Worker message-passing interface. Like OO API #1,
   this is an optional component, offering one of any number of
   potential implementations for such an API.
-    - `sqlite3-worker1.js`\  
+    - **`sqlite3-worker1.js`**\  
       Is not part of the amalgamated sources and is intended to be
       loaded by a client Worker thread. It loads the sqlite3 module
       and runs the Worker #1 API which is implemented in
       `sqlite3-api-worker1.js`.
-    - `sqlite3-worker1-promiser.js`\  
+    - **`sqlite3-worker1-promiser.js`**\  
       Is likewise not part of the amalgamated sources and provides
       a Promise-based interface into the Worker #1 API. This is
       a far user-friendlier way to interface with databases running
       in a Worker thread.
-- `sqlite3-api-opfs.js`\  
+- **`sqlite3-api-opfs.js`**\  
   is an sqlite3 VFS implementation which supports Google Chrome's
   Origin-Private FileSystem (OPFS) as a storage layer to provide
   persistent storage for database files in a browser. It requires...
-    - `sqlite3-opfs-async-proxy.js`\  
+    - **`sqlite3-opfs-async-proxy.js`**\  
       is the asynchronous backend part of the OPFS proxy. It speaks
       directly to the (async) OPFS API and channels those results back
       to its synchronous counterpart. This file, because it must be
diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js
index cace6ed51..225869794 100644
--- a/ext/wasm/api/extern-post-js.js
+++ b/ext/wasm/api/extern-post-js.js
@@ -60,7 +60,7 @@ const toExportForES6 =
     initModuleState.sqlite3Dir = li.join('/') + '/';
   }
 
-  self.sqlite3InitModule = (...args)=>{
+  self.sqlite3InitModule = function ff(...args){
     //console.warn("Using replaced sqlite3InitModule()",self.location);
     return originalInit(...args).then((EmscriptenModule)=>{
       if(self.window!==self &&
@@ -76,10 +76,12 @@ const toExportForES6 =
             Emscripten details. */
         return EmscriptenModule;
       }
-      EmscriptenModule.sqlite3.scriptInfo = initModuleState;
-      //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo);
-      const f = EmscriptenModule.sqlite3.asyncPostInit;
-      delete EmscriptenModule.sqlite3.asyncPostInit;
+      const s = EmscriptenModule.sqlite3;
+      s.scriptInfo = initModuleState;
+      //console.warn("sqlite3.scriptInfo =",s.scriptInfo);
+      if(ff.__isUnderTest) s.__isUnderTest = true;
+      const f = s.asyncPostInit;
+      delete s.asyncPostInit;
       return f();
     }).catch((e)=>{
       console.error("Exception loading sqlite3 module:",e);
diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js
index bef4d91d7..0ec0fbfbe 100644
--- a/ext/wasm/api/sqlite3-api-cleanup.js
+++ b/ext/wasm/api/sqlite3-api-cleanup.js
@@ -58,8 +58,6 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build
     self.S = sqlite3;
   }
 
-  /* Clean up temporary references to our APIs... */
-  delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */;
   Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to
                               pass the sqlite3 object off to the client. */;
 }else{
diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js
index 3099cb486..5eb3b22b3 100644
--- a/ext/wasm/api/sqlite3-api-opfs.js
+++ b/ext/wasm/api/sqlite3-api-opfs.js
@@ -129,7 +129,7 @@ const installOpfsVfs = function callee(options){
     const log =    (...args)=>logImpl(2, ...args);
     const warn =   (...args)=>logImpl(1, ...args);
     const error =  (...args)=>logImpl(0, ...args);
-    const toss = function(...args){throw new Error(args.join(' '))};
+    const toss = sqlite3.util.toss;
     const capi = sqlite3.capi;
     const wasm = sqlite3.wasm;
     const sqlite3_vfs = capi.sqlite3_vfs;
@@ -191,6 +191,8 @@ const installOpfsVfs = function callee(options){
         s.count = s.time = 0;
       }
     }/*metrics*/;      
+    const opfsVfs = new sqlite3_vfs();
+    const opfsIoMethods = new sqlite3_io_methods();
     const promiseReject = function(err){
       opfsVfs.dispose();
       return promiseReject_(err);
@@ -213,8 +215,6 @@ const installOpfsVfs = function callee(options){
           ? new sqlite3_vfs(pDVfs)
           : null /* dVfs will be null when sqlite3 is built with
                     SQLITE_OS_OTHER. */;
-    const opfsVfs = new sqlite3_vfs();
-    const opfsIoMethods = new sqlite3_io_methods();
     opfsVfs.$iVersion = 2/*yes, two*/;
     opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
     opfsVfs.$mxPathname = 1024/*sure, why not?*/;
@@ -633,77 +633,6 @@ const installOpfsVfs = function callee(options){
     */
     const __openFiles = Object.create(null);
 
-    /**
-       Installs a StructBinder-bound function pointer member of the
-       given name and function in the given StructType target object.
-       It creates a WASM proxy for the given function and arranges for
-       that proxy to be cleaned up when tgt.dispose() is called.  Throws
-       on the slightest hint of error (e.g. tgt is-not-a StructType,
-       name does not map to a struct-bound member, etc.).
-
-       Returns a proxy for this function which is bound to tgt and takes
-       2 args (name,func). That function returns the same thing,
-       permitting calls to be chained.
-
-       If called with only 1 arg, it has no side effects but returns a
-       func with the same signature as described above.
-    */
-    const installMethod = function callee(tgt, name, func){
-      if(!(tgt instanceof sqlite3.StructBinder.StructType)){
-        toss("Usage error: target object is-not-a StructType.");
-      }
-      if(1===arguments.length){
-        return (n,f)=>callee(tgt,n,f);
-      }
-      if(!callee.argcProxy){
-        callee.argcProxy = function(func,sig){
-          return function(...args){
-            if(func.length!==arguments.length){
-              toss("Argument mismatch. Native signature is:",sig);
-            }
-            return func.apply(this, args);
-          }
-        };
-        callee.removeFuncList = function(){
-          if(this.ondispose.__removeFuncList){
-            this.ondispose.__removeFuncList.forEach(
-              (v,ndx)=>{
-                if('number'===typeof v){
-                  try{wasm.uninstallFunction(v)}
-                  catch(e){/*ignore*/}
-                }
-                /* else it's a descriptive label for the next number in
-                   the list. */
-              }
-            );
-            delete this.ondispose.__removeFuncList;
-          }
-        };
-      }/*static init*/
-      const sigN = tgt.memberSignature(name);
-      if(sigN.length<2){
-        toss("Member",name," is not a function pointer. Signature =",sigN);
-      }
-      const memKey = tgt.memberKey(name);
-      const fProxy = 0
-      /** This middle-man proxy is only for use during development, to
-          confirm that we always pass the proper number of
-          arguments. We know that the C-level code will always use the
-          correct argument count. */
-            ? callee.argcProxy(func, sigN)
-            : func;
-      const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true));
-      tgt[memKey] = pFunc;
-      if(!tgt.ondispose) tgt.ondispose = [];
-      if(!tgt.ondispose.__removeFuncList){
-        tgt.ondispose.push('ondispose.__removeFuncList handler',
-                           callee.removeFuncList);
-        tgt.ondispose.__removeFuncList = [];
-      }
-      tgt.ondispose.__removeFuncList.push(memKey, pFunc);
-      return (n,f)=>callee(tgt, n, f);
-    }/*installMethod*/;
-
     const opTimer = Object.create(null);
     opTimer.op = undefined;
     opTimer.start = undefined;
@@ -729,7 +658,11 @@ const installOpfsVfs = function callee(options){
            success) release a sync-handle for it, but doing so would
            involve an inherent race condition. For the time being,
            pending a better solution, we simply report whether the
-           given pFile instance has a lock.
+           given pFile is open.
+
+           FIXME: we need to ask the async half whether a lock is
+           held, as it's possible (since long after this method was
+           implemented) that we do not hold a lock on an OPFS file.
         */
         const f = __openFiles[pFile];
         wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32');
@@ -948,14 +881,6 @@ const installOpfsVfs = function callee(options){
       };
     }
 
-    /* Install the vfs/io_methods into their C-level shared instances... */
-    for(let k of Object.keys(ioSyncWrappers)){
-      installMethod(opfsIoMethods, k, ioSyncWrappers[k]);
-    }
-    for(let k of Object.keys(vfsSyncWrappers)){
-      installMethod(opfsVfs, k, vfsSyncWrappers[k]);
-    }
-
     /**
        Expects an OPFS file path. It gets resolved, such that ".."
        components are properly expanded, and returned. If the 2nd arg
@@ -1095,8 +1020,9 @@ const installOpfsVfs = function callee(options){
        Irrevocably deletes _all_ files in the current origin's OPFS.
        Obviously, this must be used with great caution. It may throw
        an exception if removal of anything fails (e.g. a file is
-       locked), but the precise conditions under which it will throw
-       are not documented (so we cannot tell you what they are).
+       locked), but the precise conditions under which the underlying
+       APIs will throw are not documented (so we cannot tell you what
+       they are).
     */
     opfsUtil.rmfr = async function(){
       const dir = opfsUtil.rootDirectory, opt = {recurse: true};
@@ -1320,14 +1246,10 @@ const installOpfsVfs = function callee(options){
               and has finished initializing, so the real work can
               begin...*/
             try {
-              const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0);
-              if(rc){
-                toss("sqlite3_vfs_register(OPFS) failed with rc",rc);
-              }
-              if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){
-                toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
-              }
-              capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
+              sqlite3.VfsHelper.installVfs({
+                io: {struct: opfsIoMethods, methods: ioSyncWrappers},
+                vfs: {struct: opfsVfs, methods: vfsSyncWrappers}
+              });
               state.sabOPView = new Int32Array(state.sabOP);
               state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
               state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js
index 726f47e18..6d3958196 100644
--- a/ext/wasm/api/sqlite3-api-prologue.js
+++ b/ext/wasm/api/sqlite3-api-prologue.js
@@ -612,8 +612,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     isBindableTypedArray,
     isInt32, isSQLableTypedArray, isTypedArray, 
     typedArrayToString,
-    isUIThread: ()=>'undefined'===typeof WorkerGlobalScope,
+    isUIThread: ()=>(self.window===self && !!self.document),
+    // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope
     isSharedTypedArray,
+    toss: function(...args){throw new Error(args.join(' '))},
+    toss3,
     typedArrayPart
   };
     
@@ -1460,7 +1463,17 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
       //let p = lip.shift();
       //while(lip.length) p = p.then(lip.shift());
       //return p.then(()=>sqlite3);
-      return Promise.all(lip).then(()=>sqlite3);
+      return Promise.all(lip).then(()=>{
+        if(!sqlite3.__isUnderTest){
+          /* Delete references to internal-only APIs which are used by
+             some initializers. Retain them when running in test mode
+             so that we can add tests for them. */
+          delete sqlite3.util;
+          delete sqlite3.VfsHelper;
+          delete sqlite3.StructBinder;
+        }
+        return sqlite3;
+      });
     },
     /**
        scriptInfo ideally gets injected into this object by the
diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js
new file mode 100644
index 000000000..9a15dd85f
--- /dev/null
+++ b/ext/wasm/api/sqlite3-vfs-helper.js
@@ -0,0 +1,209 @@
+/*
+** 2022-11-30
+**
+** The author disclaims copyright to this source code.  In place of a
+** legal notice, here is a blessing:
+**
+** *   May you do good and not evil.
+** *   May you find forgiveness for yourself and forgive others.
+** *   May you share freely, never taking more than you give.
+*/
+
+/**
+   This file installs sqlite.VfsHelper, an object which exists
+   to assist in the creation of JavaScript implementations of
+   sqlite3_vfs. It is NOT part of the public API, and is an
+   internal implemenation detail for use in this project's
+   own development of VFSes. It may be exposed to clients
+   at some point, provided there is value in doing so.
+*/
+'use strict';
+self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
+  const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss;
+  const vh = Object.create(null);
+
+  /**
+     Does nothing more than holds a permanent reference to each
+     argument. This is useful in some cases to ensure that, e.g., a
+     custom sqlite3_io_methods instance does not get
+     garbage-collected.
+
+     Returns this object.
+  */
+  vh.holdReference = function(...args){
+    for(const v of args) this.refs.add(v);
+    return vh;
+  }.bind({refs: new Set});
+  
+  /**
+     Installs a StructBinder-bound function pointer member of the
+     given name and function in the given StructType target object.
+     It creates a WASM proxy for the given function and arranges for
+     that proxy to be cleaned up when tgt.dispose() is called.  Throws
+     on the slightest hint of error, e.g. tgt is-not-a StructType,
+     name does not map to a struct-bound member, etc.
+
+     If applyArgcCheck is true then each method gets wrapped in a
+     proxy which asserts that it is passed the expected number of
+     arguments, throwing if the argument count does not match
+     expectations. That is only recommended for dev-time usage for
+     sanity checking. Once a VFS implementation is known to be
+     working, it is a given that the C API will never call it with the
+     wrong argument count.
+
+     Returns a proxy for this function which is bound to tgt and takes
+     2 args (name,func). That function returns the same thing,
+     permitting calls to be chained.
+
+     If called with only 1 arg, it has no side effects but returns a
+     func with the same signature as described above.
+
+     If tgt.ondispose is set before this is called then it _must_
+     be an array, to which this function will append entries.
+  */
+  vh.installMethod = function callee(tgt, name, func,
+                                     applyArgcCheck=callee.installMethodArgcCheck){
+    if(!(tgt instanceof sqlite3.StructBinder.StructType)){
+      toss("Usage error: target object is-not-a StructType.");
+    }
+    if(1===arguments.length){
+      return (n,f)=>callee(tgt, n, f, applyArgcCheck);
+    }
+    if(!callee.argcProxy){
+      callee.argcProxy = function(func,sig){
+        return function(...args){
+          if(func.length!==arguments.length){
+            toss("Argument mismatch. Native signature is:",sig);
+          }
+          return func.apply(this, args);
+        }
+      };
+      /* An ondispose() callback for use with
+         sqlite3.StructBinder-created types. */
+      callee.removeFuncList = function(){
+        if(this.ondispose.__removeFuncList){
+          this.ondispose.__removeFuncList.forEach(
+            (v,ndx)=>{
+              if('number'===typeof v){
+                try{wasm.uninstallFunction(v)}
+                catch(e){/*ignore*/}
+              }
+              /* else it's a descriptive label for the next number in
+                 the list. */
+            }
+          );
+          delete this.ondispose.__removeFuncList;
+        }
+      };
+    }/*static init*/
+    const sigN = tgt.memberSignature(name);
+    if(sigN.length<2){
+      toss("Member",name," is not a function pointer. Signature =",sigN);
+    }
+    const memKey = tgt.memberKey(name);
+    const fProxy = applyArgcCheck
+    /** This middle-man proxy is only for use during development, to
+        confirm that we always pass the proper number of
+        arguments. We know that the C-level code will always use the
+        correct argument count. */
+          ? callee.argcProxy(func, sigN)
+          : func;
+    const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true));
+    tgt[memKey] = pFunc;
+    if(!tgt.ondispose) tgt.ondispose = [];
+    if(!tgt.ondispose.__removeFuncList){
+      tgt.ondispose.push('ondispose.__removeFuncList handler',
+                         callee.removeFuncList);
+      tgt.ondispose.__removeFuncList = [];
+    }
+    tgt.ondispose.__removeFuncList.push(memKey, pFunc);
+    return (n,f)=>callee(tgt, n, f, applyArgcCheck);
+  }/*installMethod*/;
+  vh.installMethod.installMethodArgcCheck = false;
+
+  /**
+     Installs methods into the given StructType-type object. Each
+     entry in the given methods object must map to a known member of
+     the given StructType, else an exception will be triggered.
+     See installMethod() for more details, including the semantics
+     of the 3rd argument.
+
+     On success, passes its first argument to holdRefence() and
+     returns this object. Throws on error.
+  */
+  vh.installMethods = function(structType, methods,
+                               applyArgcCheck=vh.installMethod.installMethodArgcCheck){
+    for(const k of Object.keys(methods)){
+      vh.installMethod(structType, k, methods[k], applyArgcCheck);
+    }
+    return vh.holdReference(structType);
+  };
+
+  /**
+     Uses sqlite3_vfs_register() to register the
+     sqlite3.capi.sqlite3_vfs-type vfs, which must have already been
+     filled out properly. If the 2nd argument is truthy, the VFS is
+     registered as the default VFS, else it is not.
+
+     On success, passes its first argument to this.holdReference() and
+     returns this object. Throws on error.
+  */
+  vh.registerVfs = function(vfs, asDefault=false){
+    if(!(vfs instanceof sqlite3.capi.sqlite3_vfs)){
+      toss("Expecting a sqlite3_vfs-type argument.");
+    }
+    const rc = capi.sqlite3_vfs_register(vfs.pointer, asDefault ? 1 : 0);
+    if(rc){
+      toss("sqlite3_vfs_register(",vfs,") failed with rc",rc);
+    }
+    if(vfs.pointer !== capi.sqlite3_vfs_find(vfs.$zName)){
+      toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS",
+           vfs);
+    }
+    return vh.holdReference(vfs);
+  };
+
+  /**
+     A wrapper for installMethods() or registerVfs() to reduce
+     installation of a VFS and/or its I/O methods to a single
+     call.
+
+     Accepts an object which contains the properties "io" and/or
+     "vfs", each of which is itself an object with following properties:
+
+     - `struct`: an sqlite3.StructType-type struct. This must be a
+       populated (except for the methods) object of type
+       sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the
+       "vfs" entry).
+
+     - `methods`: an object mapping sqlite3_io_methods method names
+       (e.g. 'xClose') to JS implementations of those methods.
+
+     For each of those object, this function passes its (`struct`,
+     `methods`, (optional) `applyArgcCheck`) properties to
+     this.installMethods().
+
+     If the `vfs` entry is set, its `struct` property is passed
+     to this.registerVfs(). The `vfs` entry may optionally have
+     an `asDefault` property, which gets passed as the 2nd
+     argument to registerVfs().
+
+     On success returns this object. Throws on error.
+  */
+  vh.installVfs = function(opt){
+    let count = 0;
+    for(const key of ['io','vfs']){
+      const o = opt[key];
+      if(o){
+        ++count;
+        this.installMethods(o.struct, o.methods, !!o.applyArgcCheck);
+        if('vfs'===key) this.registerVfs(o.struct, !!o.asDefault);
+      }
+    }
+    if(!count) toss("Misue: installVfs() options object requires at least",
+                    "one of 'io' or 'vfs' properties.");
+    return this;
+  };
+  
+  sqlite3.VfsHelper = vh;
+}/*sqlite3ApiBootstrap.initializers.push()*/);