Add the remaining vfs/io_methods wrappers to the OPFS sync/async proxy, but most are not yet tested.

FossilOrigin-Name: 44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04

1 files changed, 286 insertions, 245 deletions

diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js
index 98e268814..88fdfa603 100644
--- a/ext/wasm/sqlite3-opfs-async-proxy.js
+++ b/ext/wasm/sqlite3-opfs-async-proxy.js
@@ -20,267 +20,308 @@
   https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
 
   for demonstrating how to use the OPFS APIs.
+
+  This file is to be loaded as a Worker. It does not have any direct
+  access to the sqlite3 JS/WASM bits, so any bits which it needs (most
+  notably SQLITE_xxx integer codes) have to be imported into it via an
+  initialization process.
 */
 'use strict';
-(function(){
-  const toss = function(...args){throw new Error(args.join(' '))};
-  if(self.window === self){
-    toss("This code cannot run from the main thread.",
-         "Load it as a Worker from a separate Worker.");
-  }else if(!navigator.storage.getDirectory){
-    toss("This API requires navigator.storage.getDirectory.");
-  }
-  const logPrefix = "OPFS worker:";
-  const log = (...args)=>{
-    console.log(logPrefix,...args);
-  };
-  const warn =  (...args)=>{
-    console.warn(logPrefix,...args);
-  };
-  const error =  (...args)=>{
-    console.error(logPrefix,...args);
-  };
+const toss = function(...args){throw new Error(args.join(' '))};
+if(self.window === self){
+  toss("This code cannot run from the main thread.",
+       "Load it as a Worker from a separate Worker.");
+}else if(!navigator.storage.getDirectory){
+  toss("This API requires navigator.storage.getDirectory.");
+}
+/**
+   Will hold state copied to this object from the syncronous side of
+   this API.
+*/
+const state = Object.create(null);
+/**
+   verbose:
 
-  warn("This file is very much experimental and under construction.",self.location.pathname);
-  const wMsg = (type,payload)=>postMessage({type,payload});
+   0 = no logging output
+   1 = only errors
+   2 = warnings and errors
+   3 = debug, warnings, and errors
+*/
+state.verbose = 2;
 
-  const state = Object.create(null);
-  /*state.opSab;
-  state.sabIO;
-  state.opBuf;
-  state.opIds;
-  state.rootDir;*/
-  /**
-     Map of sqlite3_file pointers (integers) to metadata related to a
-     given OPFS file handles. The pointers are, in this side of the
-     interface, opaque file handle IDs provided by the synchronous
-     part of this constellation. Each value is an object with a structure
-     demonstrated in the xOpen() impl.
-  */
-  state.openFiles = Object.create(null);
+const __logPrefix = "OPFS asyncer:";
+const log = (...args)=>{
+  if(state.verbose>2) console.log(__logPrefix,...args);
+};
+const warn =  (...args)=>{
+  if(state.verbose>1) console.warn(__logPrefix,...args);
+};
+const error =  (...args)=>{
+  if(state.verbose) console.error(__logPrefix,...args);
+};
 
-  /**
-     Map of dir names to FileSystemDirectoryHandle objects.
-  */
-  state.dirCache = new Map;
+warn("This file is very much experimental and under construction.",self.location.pathname);
 
-  const __splitPath = (absFilename)=>{
-    const a = absFilename.split('/').filter((v)=>!!v);
-    return [a, a.pop()];
-  };
-  /**
-     Takes the absolute path to a filesystem element. Returns an array
-     of [handleOfContainingDir, filename]. If the 2nd argument is
-     truthy then each directory element leading to the file is created
-     along the way. Throws if any creation or resolution fails.
-  */
-  const getDirForPath = async function f(absFilename, createDirs = false){
-    const url = new URL(
-      absFilename, 'file://xyz'
-    ) /* use URL to resolve path pieces such as a/../b */;
-    const [path, filename] = __splitPath(url.pathname);
-    const allDirs = path.join('/');
-    let dh = state.dirCache.get(allDirs);
-    if(!dh){
-      dh = state.rootDir;
-      for(const dirName of path){
-        if(dirName){
-          dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
-        }
+/**
+   Map of sqlite3_file pointers (integers) to metadata related to a
+   given OPFS file handles. The pointers are, in this side of the
+   interface, opaque file handle IDs provided by the synchronous
+   part of this constellation. Each value is an object with a structure
+   demonstrated in the xOpen() impl.
+*/
+const __openFiles = Object.create(null);
+
+/**
+   Map of dir names to FileSystemDirectoryHandle objects.
+*/
+const __dirCache = new Map;
+
+/**
+   Takes the absolute path to a filesystem element. Returns an array
+   of [handleOfContainingDir, filename]. If the 2nd argument is
+   truthy then each directory element leading to the file is created
+   along the way. Throws if any creation or resolution fails.
+*/
+const getDirForPath = async function f(absFilename, createDirs = false){
+  const url = new URL(
+    absFilename, 'file://xyz'
+  ) /* use URL to resolve path pieces such as a/../b */;
+  const path = url.pathname.split('/').filter((v)=>!!v);
+  const filename = path.pop();
+  const allDirs = '/'+path.join('/');
+  let dh = __dirCache.get(allDirs);
+  if(!dh){
+    dh = state.rootDir;
+    for(const dirName of path){
+      if(dirName){
+        dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
       }
-      state.dirCache.set(allDirs, dh);
     }
-    return [dh, filename];
-  };
+    __dirCache.set(allDirs, dh);
+  }
+  return [dh, filename];
+};
 
-  
-  /**
-     Generates a random ASCII string len characters long, intended for
-     use as a temporary file name.
-  */
-  const randomFilename = function f(len=16){
-    if(!f._chars){
-      f._chars = "abcdefghijklmnopqrstuvwxyz"+
-        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
-        "012346789";
-      f._n = f._chars.length;
-    }
-    const a = [];
-    let i = 0;
-    for( ; i < len; ++i){
-      const ndx = Math.random() * (f._n * 64) % f._n | 0;
-      a[i] = f._chars[ndx];
-    }
-    return a.join('');
-  };
 
-  const storeAndNotify = (opName, value)=>{
-    log(opName+"() is notify()ing w/ value:",value);
-    Atomics.store(state.opBuf, state.opIds[opName], value);
-    Atomics.notify(state.opBuf, state.opIds[opName]);
-  };
+/**
+   Stores the given value at the array index reserved for the given op
+   and then Atomics.notify()'s it.
+*/
+const storeAndNotify = (opName, value)=>{
+  log(opName+"() is notify()ing w/ value:",value);
+  Atomics.store(state.opBuf, state.opIds[opName], value);
+  Atomics.notify(state.opBuf, state.opIds[opName]);
+};
 
-  const isInt32 = function(n){
-    return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
-      && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
-  };
-  const affirm32Bits = function(n){
-    return isInt32(n) || toss("Number is too large (>31 bits):",n);
-  };
+const isInt32 = function(n){
+  return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
+    && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
+};
+const affirm32Bits = function(n){
+  return isInt32(n) || toss("Number is too large (>31 bits) (FIXME!):",n);
+};
 
-  const ioMethods = {
-    xAccess: async function({filename, exists, readWrite}){
-      log("xAccess(",arguments,")");
-      const rc = 1;
-      storeAndNotify('xAccess', rc);
-    },
-    xClose: async function(fid){
-      const opName = 'xClose';
-      log(opName+"(",arguments[0],")");
-      log("state.openFiles",state.openFiles);
-      const fh = state.openFiles[fid];
-      if(fh){
-        delete state.openFiles[fid];
-        //await fh.close();
-        if(fh.accessHandle) await fh.accessHandle.close();
-        if(fh.deleteOnClose){
-          try{
-            await fh.dirHandle.removeEntry(fh.filenamePart);
-          }
-          catch(e){
-            warn("Ignoring dirHandle.removeEntry() failure of",fh);
-          }
-        }
-        log("state.openFiles",state.openFiles);
-        storeAndNotify(opName, 0);
-      }else{
-        storeAndNotify(opName, state.errCodes.NotFound);
-      }
-    },
-    xDelete: async function(filename){
-      log("xDelete(",arguments,")");
-      storeAndNotify('xClose', 0);
-    },
-    xFileSize: async function(fid){
-      log("xFileSize(",arguments,")");
-      const fh = state.openFiles[fid];
-      const sz = await fh.getSize();
-      affirm32Bits(sz);
-      storeAndNotify('xFileSize', sz | 0);
-    },
-    xOpen: async function({
-      fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/,
-      filename,
-      fileType = undefined /*mainDb, mainJournal, etc.*/,
-      create = false, readOnly = false, deleteOnClose = false,
-    }){
-      const opName = 'xOpen';
-      try{
-        if(create) readOnly = false;
-        log(opName+"(",arguments[0],")");
-
-        let hDir, filenamePart, hFile;
-        try {
-          [hDir, filenamePart] = await getDirForPath(filename, !!create);
-        }catch(e){
-          storeAndNotify(opName, state.errCodes.NotFound);
-          return;
-        }
-        hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
-        log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
-        const fobj = state.openFiles[fid] = Object.create(null);
-        fobj.filenameAbs = filename;
-        fobj.filenamePart = filenamePart;
-        fobj.dirHandle = hDir;
-        fobj.fileHandle = hFile;
-        fobj.accessHandle = undefined;
-        fobj.fileType = fileType;
-        fobj.sab = sab;
-        fobj.create = !!create;
-        fobj.readOnly = !!readOnly;
-        fobj.deleteOnClose = !!deleteOnClose;
+/**
+   Throws if fh is a file-holding object which is flagged as read-only.
+*/
+const affirmNotRO = function(opName,fh){
+  if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
+};
 
-        /**
-           wa-sqlite, at this point, grabs a SyncAccessHandle and
-           assigns it to the accessHandle prop of the file state
-           object, but it's unclear why it does that.
-        */
-        storeAndNotify(opName, 0);
+/**
+   Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
+   methods. Maintenance reminder: members are in alphabetical order
+   to simplify finding them.
+*/
+const vfsAsyncImpls = {
+  xAccess: async function({filename, exists, readWrite}){
+    warn("xAccess(",arguments[0],") is TODO");
+    const rc = state.sq3Codes.SQLITE_IOERR;
+    storeAndNotify('xAccess', rc);
+  },
+  xClose: async function(fid){
+    const opName = 'xClose';
+    log(opName+"(",arguments[0],")");
+    const fh = __openFiles[fid];
+    if(fh){
+      delete __openFiles[fid];
+      if(fh.accessHandle) await fh.accessHandle.close();
+      if(fh.deleteOnClose){
+        try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
+        catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
+      }
+      storeAndNotify(opName, 0);
+    }else{
+      storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND);
+    }
+  },
+  xDelete: async function({filename, syncDir/*ignored*/}){
+    log("xDelete(",arguments[0],")");
+    try {
+      const [hDir, filenamePart] = await getDirForPath(filename, false);
+      await hDir.removeEntry(filenamePart);
+    }catch(e){
+      /* Ignoring: _presumably_ the file can't be found. */
+    }
+    storeAndNotify('xDelete', 0);
+  },
+  xFileSize: async function(fid){
+    log("xFileSize(",arguments,")");
+    const fh = __openFiles[fid];
+    let sz;
+    try{
+      sz = await fh.accessHandle.getSize();
+      fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
+      sz = 0;
+    }catch(e){
+      error("xFileSize():",e, fh);
+      sz = state.sq3Codes.SQLITE_IOERR;
+    }
+    storeAndNotify('xFileSize', sz);
+  },
+  xOpen: async function({
+    fid/*sqlite3_file pointer*/,
+    sab/*file-specific SharedArrayBuffer*/,
+    filename,
+    fileType = undefined /*mainDb, mainJournal, etc.*/,
+    create = false, readOnly = false, deleteOnClose = false
+  }){
+    const opName = 'xOpen';
+    try{
+      if(create) readOnly = false;
+      log(opName+"(",arguments[0],")");
+      let hDir, filenamePart;
+      try {
+        [hDir, filenamePart] = await getDirForPath(filename, !!create);
       }catch(e){
-        error(opName,e);
-        storeAndNotify(opName, state.errCodes.IO);
+        storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
+        return;
       }
-    },
-    xRead: async function({fid,n,offset}){
-      log("xRead(",arguments,")");
-      affirm32Bits(n + offset);
-      const fh = state.openFiles[fid];
-      storeAndNotify('xRead',fid);
-    },
-    xSleep: async function f({ms}){
-      log("xSleep(",arguments[0],")");
-      await new Promise((resolve)=>{
-        setTimeout(()=>resolve(), ms);
-      }).finally(()=>storeAndNotify('xSleep',0));
-    },
-    xSync: async function({fid}){
-      log("xSync(",arguments,")");
-      const fh = state.openFiles[fid];
-      await fh.flush();
-      storeAndNotify('xSync',fid);
-    },
-    xTruncate: async function({fid,size}){
-      log("xTruncate(",arguments,")");
-      affirm32Bits(size);
-      const fh = state.openFiles[fid];
-      fh.truncate(size);
-      storeAndNotify('xTruncate',fid);
-    },
-    xWrite: async function({fid,src,n,offset}){
-      log("xWrite(",arguments,")");
-      const fh = state.openFiles[fid];
-      storeAndNotify('xWrite',fid);
+      const hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
+      log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
+      const fobj = __openFiles[fid] = Object.create(null);
+      fobj.filenameAbs = filename;
+      fobj.filenamePart = filenamePart;
+      fobj.dirHandle = hDir;
+      fobj.fileHandle = hFile;
+      fobj.fileType = fileType;
+      fobj.sab = sab;
+      fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
+      fobj.create = !!create;
+      fobj.readOnly = !!readOnly;
+      fobj.deleteOnClose = !!deleteOnClose;
+      /**
+         wa-sqlite, at this point, grabs a SyncAccessHandle and
+         assigns it to the accessHandle prop of the file state
+         object, but only for certain cases and it's unclear why it
+         places that limitation on it.
+      */
+      fobj.accessHandle = await hFile.createSyncAccessHandle();
+      storeAndNotify(opName, 0);
+    }catch(e){
+      error(opName,e);
+      storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
     }
-  };
-  
-  const onReady = function(){
-    self.onmessage = async function({data}){
-      log("self.onmessage",data);
-      switch(data.type){
-          case 'init':{
-            const opt = data.payload;
-            state.opSab = opt.opSab;
-            state.opBuf = new Int32Array(state.opSab);
-            state.opIds = opt.opIds;
-            state.errCodes = opt.errCodes;
-            state.sq3Codes = opt.sq3Codes;
-            Object.keys(ioMethods).forEach((k)=>{
-              if(!state.opIds[k]){
-                toss("Maintenance required: missing state.opIds[",k,"]");
-              }
-            });
-            log("init state",state);
-            break;
-          }
-          default:{
-            const m = ioMethods[data.type] || toss("Unknown message type:",data.type);
-            try {
-              await m(data.payload);
-            }catch(e){
-              error("Error handling",data.type+"():",e);
-              storeAndNotify(data.type, -99);
+  },
+  xRead: async function({fid,n,offset}){
+    log("xRead(",arguments[0],")");
+    let rc = 0;
+    const fh = __openFiles[fid];
+    try{
+      const aRead = new Uint8array(fh.sab, n);
+      const nRead = fh.accessHandle.read(aRead, {at: offset});
+      if(nRead < n){/* Zero-fill remaining bytes */
+        new Uint8array(fh.sab).fill(0, nRead, n);
+        rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
+      }
+    }catch(e){
+      error("xRead() failed",e,fh);
+      rc = state.sq3Codes.SQLITE_IOERR_READ;
+    }
+    storeAndNotify('xRead',rc);
+  },
+  xSleep: async function f(ms){
+    log("xSleep(",ms,")");
+    await new Promise((resolve)=>{
+      setTimeout(()=>resolve(), ms);
+    }).finally(()=>storeAndNotify('xSleep',0));
+  },
+  xSync: async function({fid,flags/*ignored*/}){
+    log("xSync(",arguments[0],")");
+    const fh = __openFiles[fid];
+    if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush();
+    storeAndNotify('xSync',0);
+  },
+  xTruncate: async function({fid,size}){
+    log("xTruncate(",arguments[0],")");
+    let rc = 0;
+    const fh = __openFiles[fid];
+    try{
+      affirmNotRO('xTruncate', fh);
+      await fh.accessHandle.truncate(size);
+    }catch(e){
+      error("xTruncate():",e,fh);
+      rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
+    }
+    storeAndNotify('xTruncate',rc);
+  },
+  xWrite: async function({fid,src,n,offset}){
+    log("xWrite(",arguments[0],")");
+    let rc;
+    try{
+      const fh = __openFiles[fid];
+      affirmNotRO('xWrite', fh);
+      const nOut = fh.accessHandle.write(new UInt8Array(fh.sab, 0, n), {at: offset});
+      rc = (nOut===n) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
+    }catch(e){
+      error("xWrite():",e,fh);
+      rc = state.sq3Codes.SQLITE_IOERR_WRITE;
+    }
+    storeAndNotify('xWrite',rc);
+  }
+};
+
+navigator.storage.getDirectory().then(function(d){
+  const wMsg = (type)=>postMessage({type});
+  state.rootDir = d;
+  log("state.rootDir =",state.rootDir);
+  self.onmessage = async function({data}){
+    log("self.onmessage()",data);
+    switch(data.type){
+        case 'init':{
+          /* Receive shared state from synchronous partner */
+          const opt = data.payload;
+          state.verbose = opt.verbose ?? 2;
+          state.fileBufferSize = opt.fileBufferSize;
+          state.fbInt64Offset = opt.fbInt64Offset;
+          state.opSab = opt.opSab;
+          state.opBuf = new Int32Array(state.opSab);
+          state.opIds = opt.opIds;
+          state.sq3Codes = opt.sq3Codes;
+          Object.keys(vfsAsyncImpls).forEach((k)=>{
+            if(!Number.isFinite(state.opIds[k])){
+              toss("Maintenance required: missing state.opIds[",k,"]");
             }
-            break;
+          });
+          log("init state",state);
+          wMsg('inited');
+          break;
+        }
+        default:{
+          let err;
+          const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type);
+          try {
+            await m(data.payload).catch((e)=>err=e);
+          }catch(e){
+            err = e;
           }
-      }
-    };      
-    wMsg('ready');
+          if(err){
+            error("Error handling",data.type+"():",e);
+            storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR);
+          }
+          break;
+        }
+    }
   };
-
-  navigator.storage.getDirectory().then(function(d){
-    state.rootDir = d;
-    log("state.rootDir =",state.rootDir);
-    onReady();
-  });
-    
-})();
+  wMsg('loaded');
+});